From 2637033899be75214f0cf3c28c64fa6c9f397b8f Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Sat, 15 Feb 2014 14:07:22 -0600 Subject: [PATCH 01/42] Add dict support for bound statements --- cassandra/query.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cassandra/query.py b/cassandra/query.py index 698eb5d6..e187b794 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -220,7 +220,26 @@ class BoundStatement(Statement): sequence, even if you are only binding one value. """ col_meta = self.prepared_statement.column_metadata - if len(values) > len(col_meta): + + dict_values = [] + if isinstance(values, dict): + dict_values = values + arranged_values = [] + for col in col_meta: + arranged_values.append(values[col[2]]) + values = arranged_values + + if len(values) > len(col_meta) or len(dict_values) > len(col_meta): + columns = set() + for col in col_meta: + columns.add(col[2]) + if dict_values: + difference = set(dict_values.keys()).difference(columns) + msg = "Too many arguments provided to bind() (got %d, expected %d). " + \ + "Unexpected keys %s" + msg = msg % (len(values), len(col_meta), difference) + raise ValueError(msg) + raise ValueError( "Too many arguments provided to bind() (got %d, expected %d)" % (len(values), len(col_meta))) From 325d42e6e6213ce91f60707fe8dcafa8bcf459e3 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Sat, 15 Feb 2014 14:15:31 -0600 Subject: [PATCH 02/42] Better logging for dict KeyErrors --- cassandra/query.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cassandra/query.py b/cassandra/query.py index e187b794..d5e84ec6 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -226,7 +226,13 @@ class BoundStatement(Statement): dict_values = values arranged_values = [] for col in col_meta: - arranged_values.append(values[col[2]]) + try: + arranged_values.append(values[col[2]]) + except KeyError: + raise KeyError( + 'Column name `%s` not found in bound dict.' % + (col[2])) + values = arranged_values if len(values) > len(col_meta) or len(dict_values) > len(col_meta): From 98a55550302fc0634b72a09ebec20b4b4a371799 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Sat, 15 Feb 2014 14:16:55 -0600 Subject: [PATCH 03/42] Missing period --- cassandra/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/query.py b/cassandra/query.py index d5e84ec6..b5ff0916 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -242,7 +242,7 @@ class BoundStatement(Statement): if dict_values: difference = set(dict_values.keys()).difference(columns) msg = "Too many arguments provided to bind() (got %d, expected %d). " + \ - "Unexpected keys %s" + "Unexpected keys %s." msg = msg % (len(values), len(col_meta), difference) raise ValueError(msg) From 49ee7cb4a1d804d83eed42248b27fbca0b4dcbb4 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Sat, 15 Feb 2014 14:25:23 -0600 Subject: [PATCH 04/42] Fix whitespace --- cassandra/query.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cassandra/query.py b/cassandra/query.py index b5ff0916..f3a479a1 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -232,7 +232,6 @@ class BoundStatement(Statement): raise KeyError( 'Column name `%s` not found in bound dict.' % (col[2])) - values = arranged_values if len(values) > len(col_meta) or len(dict_values) > len(col_meta): From f304b3d456ee854e87cc16d0319e8f1f82873252 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 20 Mar 2014 20:13:16 -0500 Subject: [PATCH 05/42] document and restructure acceptance --- cassandra/query.py | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/cassandra/query.py b/cassandra/query.py index ce993dd9..4cb1d924 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -214,35 +214,48 @@ class BoundStatement(Statement): def bind(self, values): """ Binds a sequence of values for the prepared statement parameters - and returns this instance. Note that `values` *must* be a - sequence, even if you are only binding one value. + and returns this instance. Note that `values` *must* be: + * a sequence, even if you are only binding one value, or + * a dict that relates 1-to-1 between dict keys and columns """ col_meta = self.prepared_statement.column_metadata - dict_values = [] + # special case for binding dicts if isinstance(values, dict): dict_values = values - arranged_values = [] + values = [] + + # sort values accordingly for col in col_meta: try: - arranged_values.append(values[col[2]]) + values.append(dict_values[col[2]]) except KeyError: raise KeyError( 'Column name `%s` not found in bound dict.' % (col[2])) - values = arranged_values - if len(values) > len(col_meta) or len(dict_values) > len(col_meta): - columns = set() - for col in col_meta: - columns.add(col[2]) - if dict_values: - difference = set(dict_values.keys()).difference(columns) - msg = "Too many arguments provided to bind() (got %d, expected %d). " + \ - "Unexpected keys %s." + # ensure a 1-to-1 dict keys to columns relationship + if len(dict_values) != len(col_meta): + # find expected columns + columns = set() + for col in col_meta: + columns.add(col[2]) + + # generate error message + if len(dict_values) > len(col_meta): + difference = set(dict_values.keys()).difference(columns) + msg = "Too many arguments provided to bind() (got %d, expected %d). " + \ + "Unexpected keys %s." + else: + difference = set(columns).difference(dict_values.keys()) + msg = "Too few arguments provided to bind() (got %d, expected %d). " + \ + "Expected keys %s." + + # exit with error message msg = msg % (len(values), len(col_meta), difference) raise ValueError(msg) + if len(values) > len(col_meta): raise ValueError( "Too many arguments provided to bind() (got %d, expected %d)" % (len(values), len(col_meta))) From 9ecd98171a3c7b56eeb04309e27a9e9e8a48c773 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 20 Mar 2014 21:05:14 -0500 Subject: [PATCH 06/42] Test cases for binding dicts --- .../standard/test_prepared_statements.py | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/tests/integration/standard/test_prepared_statements.py b/tests/integration/standard/test_prepared_statements.py index 4fe5e0fb..acf93ff4 100644 --- a/tests/integration/standard/test_prepared_statements.py +++ b/tests/integration/standard/test_prepared_statements.py @@ -53,6 +53,32 @@ class PreparedStatementTests(unittest.TestCase): results = session.execute(bound) self.assertEquals(results, [('a', 'b', 'c')]) + # test with new dict binding + prepared = session.prepare( + """ + INSERT INTO cf0 (a, b, c) VALUES (?, ?, ?) + """) + + self.assertIsInstance(prepared, PreparedStatement) + bound = prepared.bind({ + 'a': 'x', + 'b': 'y', + 'c': 'z' + }) + + session.execute(bound) + + prepared = session.prepare( + """ + SELECT * FROM cf0 WHERE a=? + """) + + self.assertIsInstance(prepared, PreparedStatement) + + bound = prepared.bind({'a': 'x'}) + results = session.execute(bound) + self.assertEquals(results, [('x', 'y', 'z')]) + def test_missing_primary_key(self): """ Ensure an InvalidRequest is thrown @@ -71,6 +97,25 @@ class PreparedStatementTests(unittest.TestCase): bound = prepared.bind((1,)) self.assertRaises(InvalidRequest, session.execute, bound) + def test_missing_primary_key_dicts(self): + """ + Ensure an InvalidRequest is thrown + when prepared statements are missing the primary key + with dict bindings + """ + + cluster = Cluster() + session = cluster.connect() + + prepared = session.prepare( + """ + INSERT INTO test3rf.test (v) VALUES (?) + """) + + self.assertIsInstance(prepared, PreparedStatement) + bound = prepared.bind({'v': 1}) + self.assertRaises(InvalidRequest, session.execute, bound) + def test_too_many_bind_values(self): """ Ensure a ValueError is thrown when attempting to bind too many variables @@ -87,6 +132,27 @@ class PreparedStatementTests(unittest.TestCase): self.assertIsInstance(prepared, PreparedStatement) self.assertRaises(ValueError, prepared.bind, (1,2)) + def test_too_many_bind_values_dicts(self): + """ + Ensure a ValueError is thrown when attempting to bind too many variables + with dict bindings + """ + + cluster = Cluster() + session = cluster.connect() + + prepared = session.prepare( + """ + INSERT INTO test3rf.test (v) VALUES (?) + """) + + self.assertIsInstance(prepared, PreparedStatement) + self.assertRaises(ValueError, prepared.bind, {'k': 1, 'v': 2}) + + # also catch too few variables with dicts + self.assertIsInstance(prepared, PreparedStatement) + self.assertRaises(KeyError, prepared.bind, {}) + def test_none_values(self): """ Ensure binding None is handled correctly @@ -114,6 +180,35 @@ class PreparedStatementTests(unittest.TestCase): results = session.execute(bound) self.assertEquals(results[0].v, None) + def test_none_values_dicts(self): + """ + Ensure binding None is handled correctly with dict bindings + """ + + cluster = Cluster() + session = cluster.connect() + + + # test with new dict binding + prepared = session.prepare( + """ + INSERT INTO test3rf.test (k, v) VALUES (?, ?) + """) + + self.assertIsInstance(prepared, PreparedStatement) + bound = prepared.bind({'k': 1, 'v': None}) + session.execute(bound) + + prepared = session.prepare( + """ + SELECT * FROM test3rf.test WHERE k=? + """) + self.assertIsInstance(prepared, PreparedStatement) + + bound = prepared.bind({'k': 1}) + results = session.execute(bound) + self.assertEquals(results[0].v, None) + def test_async_binding(self): """ Ensure None binding over async queries @@ -140,3 +235,31 @@ class PreparedStatementTests(unittest.TestCase): future = session.execute_async(prepared, (873,)) results = future.result() self.assertEquals(results[0].v, None) + + + def test_async_binding_dicts(self): + """ + Ensure None binding over async queries with dict bindings + """ + + cluster = Cluster() + session = cluster.connect() + + prepared = session.prepare( + """ + INSERT INTO test3rf.test (k, v) VALUES (?, ?) + """) + + self.assertIsInstance(prepared, PreparedStatement) + future = session.execute_async(prepared, {'k': 873, 'v': None}) + future.result() + + prepared = session.prepare( + """ + SELECT * FROM test3rf.test WHERE k=? + """) + self.assertIsInstance(prepared, PreparedStatement) + + future = session.execute_async(prepared, {'k': 873}) + results = future.result() + self.assertEquals(results[0].v, None) From a84a222bddf98f16944383d268ee41d16563830a Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Wed, 2 Apr 2014 18:16:58 -0500 Subject: [PATCH 07/42] Correct catch Empty exception on connection id queue Broken in master, but not in any release --- cassandra/connection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cassandra/connection.py b/cassandra/connection.py index 005f813f..4780f0c9 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -7,9 +7,9 @@ import time import traceback if 'gevent.monkey' in sys.modules: - from gevent.queue import Queue + from gevent.queue import Queue, Empty else: - from Queue import Queue # noqa + from Queue import Queue, Empty # noqa from cassandra import ConsistencyLevel, AuthenticationFailed, OperationTimedOut from cassandra.marshal import int8_unpack, int32_pack @@ -204,7 +204,7 @@ class Connection(object): if not wait_for_id: try: request_id = self._id_queue.get_nowait() - except Queue.Empty: + except Empty: raise ConnectionBusy( "Connection to %s is at the max number of requests" % self.host) else: From 667a5cab50e387fbaec450ce3f0b061c93d17f30 Mon Sep 17 00:00:00 2001 From: paul cannon Date: Thu, 3 Apr 2014 11:49:10 -0600 Subject: [PATCH 08/42] Debian packaging --- debian/changelog | 5 +++ debian/compat | 1 + debian/control | 46 ++++++++++++++++++++ debian/copyright | 28 ++++++++++++ debian/patches/0001-don-t-use-ez_setup.patch | 40 +++++++++++++++++ debian/patches/series | 1 + debian/python-cassandra-driver-dbg.install | 2 + debian/python-cassandra-driver-doc.docs | 1 + debian/python-cassandra-driver.install | 4 ++ debian/rules | 16 +++++++ debian/source/format | 1 + 11 files changed, 145 insertions(+) create mode 100644 debian/changelog create mode 100644 debian/compat create mode 100644 debian/control create mode 100644 debian/copyright create mode 100644 debian/patches/0001-don-t-use-ez_setup.patch create mode 100644 debian/patches/series create mode 100644 debian/python-cassandra-driver-dbg.install create mode 100644 debian/python-cassandra-driver-doc.docs create mode 100644 debian/python-cassandra-driver.install create mode 100755 debian/rules create mode 100644 debian/source/format diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 00000000..c98ea8c9 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +python-cassandra-driver (1.1.0~prerelease-1) unstable; urgency=low + + * Initial packaging + + -- paul cannon Thu, 03 Apr 2014 10:30:11 -0600 diff --git a/debian/compat b/debian/compat new file mode 100644 index 00000000..ec635144 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +9 diff --git a/debian/control b/debian/control new file mode 100644 index 00000000..c13f4e86 --- /dev/null +++ b/debian/control @@ -0,0 +1,46 @@ +Source: python-cassandra-driver +Maintainer: paul cannon +Section: python +Priority: optional +Build-Depends: python-all-dev (>= 2.6.6-3), python-all-dbg, debhelper (>= 9), + python-sphinx (>= 1.0.7+dfsg) | python3-sphinx, libev-dev, + python-concurrent.futures | python-futures, python-setuptools, + python-nose, python-mock, python-yaml, python-gevent, + python-blist, python-tz +X-Python-Version: >= 2.7 +Standards-Version: 3.9.4 + +Package: python-cassandra-driver +Architecture: any +Depends: ${misc:Depends}, ${python:Depends}, ${shlibs:Depends}, python-blist, + python-concurrent.futures | python-futures +Provides: ${python:Provides} +Recommends: python-scales +Suggests: python-cassandra-driver-doc +Description: Python driver for Apache Cassandra + This driver works exclusively with the Cassandra Query Language v3 (CQL3) + and Cassandra's native protocol. As such, only Cassandra 1.2+ is supported. + +Package: python-cassandra-driver-dbg +Architecture: any +Depends: ${misc:Depends}, ${python:Depends}, ${shlibs:Depends}, + python-cassandra-driver (= ${binary:Version}) +Provides: ${python:Provides} +Section: debug +Priority: extra +Description: Python driver for Apache Cassandra (debug build and symbols) + This driver works exclusively with the Cassandra Query Language v3 (CQL3) + and Cassandra's native protocol. As such, only Cassandra 1.2+ is supported. + . + This package contains debug builds of the extensions and debug symbols for + the extensions in the main package. + +Package: python-cassandra-driver-doc +Architecture: all +Section: doc +Priority: extra +Depends: ${misc:Depends}, ${sphinxdoc:Depends} +Suggests: python-cassandra-driver +Description: Python driver for Apache Cassandra (documentation) + This contains HTML documentation for the use of the Python Cassandra + driver. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 00000000..f138547a --- /dev/null +++ b/debian/copyright @@ -0,0 +1,28 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: python-driver +Upstream-Contact: Tyler Hobbs +Source: https://github.com/datastax/python-driver + +Files: * +Copyright: Copyright 2013, DataStax +License: Apache-2.0 + +Files: debian/* +Copyright: Copyright (c) 2014 by Space Monkey, Inc. +License: Apache-2.0 + +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the full text of the Apache License version 2.0 + can be found in the file `/usr/share/common-licenses/Apache-2.0'. diff --git a/debian/patches/0001-don-t-use-ez_setup.patch b/debian/patches/0001-don-t-use-ez_setup.patch new file mode 100644 index 00000000..ece0bef8 --- /dev/null +++ b/debian/patches/0001-don-t-use-ez_setup.patch @@ -0,0 +1,40 @@ +From: paul cannon +Date: Thu, 3 Apr 2014 11:27:09 -0600 +Subject: don't use ez_setup + +Debian packages aren't supposed to download stuff while building, and +since the version of setuptools in stable is less than the one ez_setup +wants, and since some system python packages don't ship their .egg-info +directories, it might try. + +It's ok though, we can rely on the Depends and Build-Depends for making +sure python-setuptools and the various other deps are around at the right +times. +--- + setup.py | 7 ++----- + 1 file changed, 2 insertions(+), 5 deletions(-) + +diff --git a/setup.py b/setup.py +index 0c28d3d..c0fd6c1 100644 +--- a/setup.py ++++ b/setup.py +@@ -1,8 +1,5 @@ + import sys + +-import ez_setup +-ez_setup.use_setuptools() +- + if __name__ == '__main__' and sys.argv[1] == "gevent_nosetests": + from gevent.monkey import patch_all + patch_all() +@@ -174,8 +171,8 @@ def run_setup(extensions): + author_email='tyler@datastax.com', + packages=['cassandra', 'cassandra.io'], + include_package_data=True, +- install_requires=dependencies, +- tests_require=['nose', 'mock', 'ccm', 'unittest2', 'PyYAML', 'pytz'], ++ install_requires=(), ++ tests_require=(), + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', diff --git a/debian/patches/series b/debian/patches/series new file mode 100644 index 00000000..25373f18 --- /dev/null +++ b/debian/patches/series @@ -0,0 +1 @@ +0001-don-t-use-ez_setup.patch diff --git a/debian/python-cassandra-driver-dbg.install b/debian/python-cassandra-driver-dbg.install new file mode 100644 index 00000000..a75d4c78 --- /dev/null +++ b/debian/python-cassandra-driver-dbg.install @@ -0,0 +1,2 @@ +usr/lib/python2*/*-packages/cassandra/*_d.so +usr/lib/python2*/*-packages/cassandra/io/*_d.so diff --git a/debian/python-cassandra-driver-doc.docs b/debian/python-cassandra-driver-doc.docs new file mode 100644 index 00000000..300da921 --- /dev/null +++ b/debian/python-cassandra-driver-doc.docs @@ -0,0 +1 @@ +docs/_build/*/* diff --git a/debian/python-cassandra-driver.install b/debian/python-cassandra-driver.install new file mode 100644 index 00000000..cf5c1ebe --- /dev/null +++ b/debian/python-cassandra-driver.install @@ -0,0 +1,4 @@ +usr/lib/python2*/*-packages/cassandra/*[!_][!_].so +usr/lib/python2*/*-packages/cassandra/*.py +usr/lib/python2*/*-packages/cassandra/io/*[!_][!_].so +usr/lib/python2*/*-packages/cassandra/io/*.py diff --git a/debian/rules b/debian/rules new file mode 100755 index 00000000..1943cfc1 --- /dev/null +++ b/debian/rules @@ -0,0 +1,16 @@ +#!/usr/bin/make -f + +%: + dh $@ --with python2,sphinxdoc + +override_dh_auto_build: + dh_auto_build + python setup.py doc + +ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) +override_dh_auto_test: + python setup.py gevent_nosetests +endif + +override_dh_strip: + dh_strip --dbg-package=python-cassandra-driver-dbg diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 00000000..163aaf8d --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) From 3115c4bf3b88b41d73a8cd9c2740010fbb732edc Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Thu, 3 Apr 2014 16:44:08 -0500 Subject: [PATCH 09/42] Add methods for concurrent execution Fixes #7 --- cassandra/concurrent.py | 148 ++++++++++++++++++ tests/integration/standard/test_concurrent.py | 113 +++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 cassandra/concurrent.py create mode 100644 tests/integration/standard/test_concurrent.py diff --git a/cassandra/concurrent.py b/cassandra/concurrent.py new file mode 100644 index 00000000..6edd54a1 --- /dev/null +++ b/cassandra/concurrent.py @@ -0,0 +1,148 @@ +import sys + +from itertools import count, cycle +from threading import Event + + +def execute_concurrent(session, statements_and_parameters, concurrency=100, raise_on_first_error=True): + """ + Executes a sequence of (statement, parameters) tuples concurrently. Each + ``parameters`` item must be a sequence or :const:`None`. + + A sequence of ``(success, result_or_exc)`` tuples is returned in the same + order that the statements were passed in. If ``success`` if :const:`False`, + there was an error executing the statement, and ``result_or_exc`` will be + an :class:`Exception`. If ``success`` is :const:`True`, ``result_or_exc`` + will be the query result. + + If `raise_on_first_error` is left as :const:`True`, execution will stop + after the first failed statement and the corresponding exception will be + raised. + + The `concurrency` parameter controls how many statements will be executed + concurrently. It is recommended that this be kept below the number of + core connections per host times the number of connected hosts (see + :meth:`.Cluster.set_core_connections_per_host`). If that amount is exceeded, + the event loop thread may attempt to block on new connection creation, + substantially impacting throughput. + + Example usage:: + + select_statement = session.prepare("SELECT * FROM users WHERE id=?") + + statements_and_params = [] + for user_id in user_ids: + statatements_and_params.append( + (select_statement, user_id)) + + results = execute_concurrent( + session, statements_and_params, raise_on_first_error=False) + + for (success, result) in results: + if not success: + handle_error(result) # result will be an Exception + else: + process_user(result[0]) # result will be a list of rows + + """ + if concurrency <= 0: + raise ValueError("concurrency must be greater than 0") + + if not statements_and_parameters: + return [] + + event = Event() + first_error = [] if raise_on_first_error else None + to_execute = len(statements_and_parameters) # TODO handle iterators/generators + results = [None] * to_execute + num_finished = count(start=1) + statements = enumerate(iter(statements_and_parameters)) + for i in xrange(min(concurrency, len(statements_and_parameters))): + _execute_next(_sentinel, i, event, session, statements, results, num_finished, to_execute, first_error) + + event.wait() + if first_error: + raise first_error[0] + else: + return results + + +def execute_concurrent_with_args(session, statement, parameters, *args, **kwargs): + """ + Like :meth:`~.execute_concurrent`, but takes a single statement and a + sequence of parameters. Each item in ``parameters`` should be a sequence + or :const:`None`. + + Example usage:: + + statement = session.prepare("INSERT INTO mytable (a, b) VALUES (1, ?)") + parameters = [(x,) for x in range(1000)] + execute_concurrent_with_args(session, statement, parameters) + """ + return execute_concurrent(session, zip(cycle((statement,)), parameters), *args, **kwargs) + + +_sentinel = object() + + +def _handle_error(error, result_index, event, session, statements, results, num_finished, to_execute, first_error): + if first_error is not None: + first_error.append(error) + event.set() + return + else: + results[result_index] = (False, error) + if num_finished.next() >= to_execute: + event.set() + return + + try: + (next_index, (statement, params)) = statements.next() + except StopIteration: + return + + args = (next_index, event, session, statements, results, num_finished, to_execute, first_error) + try: + session.execute_async(statement, params).add_callbacks( + callback=_execute_next, callback_args=args, + errback=_handle_error, errback_args=args) + except Exception as exc: + if first_error is not None: + first_error.append(sys.exc_info()) + event.set() + return + else: + results[next_index] = (False, exc) + if num_finished.next() >= to_execute: + event.set() + return + + +def _execute_next(result, result_index, event, session, statements, results, num_finished, to_execute, first_error): + if result is not _sentinel: + results[result_index] = (True, result) + finished = num_finished.next() + if finished >= to_execute: + event.set() + return + + try: + (next_index, (statement, params)) = statements.next() + except StopIteration: + return + + args = (next_index, event, session, statements, results, num_finished, to_execute, first_error) + try: + session.execute_async(statement, params).add_callbacks( + callback=_execute_next, callback_args=args, + errback=_handle_error, errback_args=args) + except Exception as exc: + if first_error is not None: + first_error.append(sys.exc_info()) + event.set() + return + else: + results[next_index] = (False, exc) + if num_finished.next() >= to_execute: + event.set() + return diff --git a/tests/integration/standard/test_concurrent.py b/tests/integration/standard/test_concurrent.py new file mode 100644 index 00000000..352033a0 --- /dev/null +++ b/tests/integration/standard/test_concurrent.py @@ -0,0 +1,113 @@ +from tests.integration import PROTOCOL_VERSION + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + +from itertools import cycle + +from cassandra import InvalidRequest +from cassandra.cluster import Cluster +from cassandra.concurrent import (execute_concurrent, + execute_concurrent_with_args) +from cassandra.policies import HostDistance +from cassandra.query import tuple_factory + + +class ClusterTests(unittest.TestCase): + + def setUp(self): + self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.cluster.set_core_connections_per_host(HostDistance.LOCAL, 1) + self.session = self.cluster.connect() + self.session.row_factory = tuple_factory + + def test_execute_concurrent(self): + for num_statements in (0, 1, 2, 7, 10, 99, 100, 101, 199, 200, 201): + # write + statements = cycle(("INSERT INTO test3rf.test (k, v) VALUES (%s, %s)", )) + parameters = [(i, i) for i in range(num_statements)] + + results = execute_concurrent(self.session, zip(statements, parameters)) + self.assertEqual(num_statements, len(results)) + self.assertEqual([(True, None)] * num_statements, results) + + # read + statements = cycle(("SELECT v FROM test3rf.test WHERE k=%s", )) + parameters = [(i, ) for i in range(num_statements)] + + results = execute_concurrent(self.session, zip(statements, parameters)) + self.assertEqual(num_statements, len(results)) + self.assertEqual([(True, [(i,)]) for i in range(num_statements)], results) + + def test_execute_concurrent_with_args(self): + for num_statements in (0, 1, 2, 7, 10, 99, 100, 101, 199, 200, 201): + statement = "INSERT INTO test3rf.test (k, v) VALUES (%s, %s)" + parameters = [(i, i) for i in range(num_statements)] + + results = execute_concurrent_with_args(self.session, statement, parameters) + self.assertEqual(num_statements, len(results)) + self.assertEqual([(True, None)] * num_statements, results) + + # read + statement = "SELECT v FROM test3rf.test WHERE k=%s" + parameters = [(i, ) for i in range(num_statements)] + + results = execute_concurrent_with_args(self.session, statement, parameters) + self.assertEqual(num_statements, len(results)) + self.assertEqual([(True, [(i,)]) for i in range(num_statements)], results) + + def test_first_failure(self): + statements = cycle(("INSERT INTO test3rf.test (k, v) VALUES (%s, %s)", )) + parameters = [(i, i) for i in range(100)] + + # we'll get an error back from the server + parameters[57] = ('efefef', 'awefawefawef') + + self.assertRaises( + InvalidRequest, + execute_concurrent, self.session, zip(statements, parameters), raise_on_first_error=True) + + def test_first_failure_client_side(self): + statements = cycle(("INSERT INTO test3rf.test (k, v) VALUES (%s, %s)", )) + parameters = [(i, i) for i in range(100)] + + # the driver will raise an error when binding the params + parameters[57] = 1 + + self.assertRaises( + TypeError, + execute_concurrent, self.session, zip(statements, parameters), raise_on_first_error=True) + + def test_no_raise_on_first_failure(self): + statements = cycle(("INSERT INTO test3rf.test (k, v) VALUES (%s, %s)", )) + parameters = [(i, i) for i in range(100)] + + # we'll get an error back from the server + parameters[57] = ('efefef', 'awefawefawef') + + results = execute_concurrent(self.session, zip(statements, parameters), raise_on_first_error=False) + for i, (success, result) in enumerate(results): + if i == 57: + self.assertFalse(success) + self.assertIsInstance(result, InvalidRequest) + else: + self.assertTrue(success) + self.assertEqual(None, result) + + def test_no_raise_on_first_failure_client_side(self): + statements = cycle(("INSERT INTO test3rf.test (k, v) VALUES (%s, %s)", )) + parameters = [(i, i) for i in range(100)] + + # the driver will raise an error when binding the params + parameters[57] = i + + results = execute_concurrent(self.session, zip(statements, parameters), raise_on_first_error=False) + for i, (success, result) in enumerate(results): + if i == 57: + self.assertFalse(success) + self.assertIsInstance(result, TypeError) + else: + self.assertTrue(success) + self.assertEqual(None, result) From 23f71e50a884386683c317bec1b3b555853aa258 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Thu, 3 Apr 2014 17:00:05 -0500 Subject: [PATCH 10/42] Use absolute imports in cassandra.cluster --- cassandra/cluster.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 807b3a1b..eb02f295 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -2,6 +2,7 @@ This module houses the main classes you will interact with, :class:`.Cluster` and :class:`.Session`. """ +from __future__ import absolute_import from concurrent.futures import ThreadPoolExecutor import logging From ff1532bf7d432e2015c97a492d7bf4f5d61b085a Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Thu, 3 Apr 2014 17:05:20 -0500 Subject: [PATCH 11/42] Make test_concurrent compatible with master --- tests/integration/standard/test_concurrent.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/integration/standard/test_concurrent.py b/tests/integration/standard/test_concurrent.py index 352033a0..e847cc4e 100644 --- a/tests/integration/standard/test_concurrent.py +++ b/tests/integration/standard/test_concurrent.py @@ -1,5 +1,3 @@ -from tests.integration import PROTOCOL_VERSION - try: import unittest2 as unittest except ImportError: @@ -12,13 +10,13 @@ from cassandra.cluster import Cluster from cassandra.concurrent import (execute_concurrent, execute_concurrent_with_args) from cassandra.policies import HostDistance -from cassandra.query import tuple_factory +from cassandra.decoder import tuple_factory class ClusterTests(unittest.TestCase): def setUp(self): - self.cluster = Cluster(protocol_version=PROTOCOL_VERSION) + self.cluster = Cluster() self.cluster.set_core_connections_per_host(HostDistance.LOCAL, 1) self.session = self.cluster.connect() self.session.row_factory = tuple_factory From 858c9358460c0b5959b9ea7abb9cfe1f19eaf5b0 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Thu, 3 Apr 2014 17:09:09 -0500 Subject: [PATCH 12/42] Update changelog --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cc4e93d5..f9db0899 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,9 @@ Features github issue #46) * Support static columns in schemas, which are available starting in Cassandra 2.1. (github issue #91) +* Add debian packaging (github issue #101) +* Add utility methods for easy concurrent execution of statements. See + the new cassandra.concurrent module. (github issue #7) Bug Fixes --------- From e76a838158603892a4255136153959615ee10ebf Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Thu, 3 Apr 2014 17:14:01 -0500 Subject: [PATCH 13/42] Docs for cassandra.concurrent --- docs/api/cassandra/concurrent.rst | 8 ++++++++ docs/api/index.rst | 1 + 2 files changed, 9 insertions(+) create mode 100644 docs/api/cassandra/concurrent.rst diff --git a/docs/api/cassandra/concurrent.rst b/docs/api/cassandra/concurrent.rst new file mode 100644 index 00000000..f4bab6f0 --- /dev/null +++ b/docs/api/cassandra/concurrent.rst @@ -0,0 +1,8 @@ +``cassandra.concurrent`` - Utilities for Concurrent Statement Execution +======================================================================= + +.. module:: cassandra.concurrent + +.. autofunction:: execute_concurrent + +.. autofunction:: execute_concurrent_with_args diff --git a/docs/api/index.rst b/docs/api/index.rst index a8284e2d..37bcb379 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -10,6 +10,7 @@ API Documentation cassandra/metadata cassandra/query cassandra/pool + cassandra/concurrent cassandra/connection cassandra/io/asyncorereactor cassandra/io/libevreactor From 944c22ad512c3125061ebd56d36d9a89dd32ceb0 Mon Sep 17 00:00:00 2001 From: Samuel Toriel Date: Mon, 7 Apr 2014 16:39:14 -0400 Subject: [PATCH 14/42] When creating a new connection use the same query When a new control connection is made a table is queried multiple times before the control connection is established. This patch combines the queries to that table by using "preloaded" results. This seemed like a minimal way to make the change, but might not necessarily be the best way. Let me know if you feel like a different approach should be used. --- cassandra/cluster.py | 100 ++++++++++++++++---------- tests/unit/test_control_connection.py | 86 +++++++++++++++++++++- 2 files changed, 146 insertions(+), 40 deletions(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index eb02f295..35d6b92e 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1322,8 +1322,8 @@ class ControlConnection(object): _SELECT_COLUMN_FAMILIES = "SELECT * FROM system.schema_columnfamilies" _SELECT_COLUMNS = "SELECT * FROM system.schema_columns" - _SELECT_PEERS = "SELECT peer, data_center, rack, tokens, rpc_address FROM system.peers" - _SELECT_LOCAL = "SELECT cluster_name, data_center, rack, tokens, partitioner FROM system.local WHERE key='local'" + _SELECT_PEERS = "SELECT peer, data_center, rack, tokens, rpc_address, schema_version FROM system.peers" + _SELECT_LOCAL = "SELECT cluster_name, data_center, rack, tokens, partitioner, schema_version FROM system.local WHERE key='local'" _SELECT_SCHEMA_PEERS = "SELECT rpc_address, schema_version FROM system.peers" _SELECT_SCHEMA_LOCAL = "SELECT schema_version FROM system.local WHERE key='local'" @@ -1406,8 +1406,13 @@ class ControlConnection(object): "SCHEMA_CHANGE": self._handle_schema_change }) - self._refresh_node_list_and_token_map(connection) - self._refresh_schema(connection) + peers_query = QueryMessage(query=self._SELECT_PEERS, consistency_level=ConsistencyLevel.ONE) + local_query = QueryMessage(query=self._SELECT_LOCAL, consistency_level=ConsistencyLevel.ONE) + shared_results = connection.wait_for_responses( + peers_query, local_query, timeout=self._timeout) + + self._refresh_node_list_and_token_map(connection, preloaded_results=shared_results) + self._refresh_schema(connection, preloaded_results=shared_results) except Exception: connection.close() raise @@ -1489,11 +1494,11 @@ class ControlConnection(object): log.debug("[control connection] Error refreshing schema", exc_info=True) self._signal_error() - def _refresh_schema(self, connection, keyspace=None, table=None): + def _refresh_schema(self, connection, keyspace=None, table=None, preloaded_results=None): if self._cluster._is_shutdown: return - self.wait_for_schema_agreement(connection) + self.wait_for_schema_agreement(connection, preloaded_results=preloaded_results) where_clause = "" if keyspace: @@ -1540,13 +1545,19 @@ class ControlConnection(object): log.debug("[control connection] Error refreshing node list and token map", exc_info=True) self._signal_error() - def _refresh_node_list_and_token_map(self, connection): - log.debug("[control connection] Refreshing node list and token map") - cl = ConsistencyLevel.ONE - peers_query = QueryMessage(query=self._SELECT_PEERS, consistency_level=cl) - local_query = QueryMessage(query=self._SELECT_LOCAL, consistency_level=cl) - peers_result, local_result = connection.wait_for_responses( - peers_query, local_query, timeout=self._timeout) + def _refresh_node_list_and_token_map(self, connection, preloaded_results=None): + if preloaded_results: + log.debug("[control connection] Refreshing node list and token map using preloaded results") + peers_result = preloaded_results[0] + local_result = preloaded_results[1] + else: + log.debug("[control connection] Refreshing node list and token map") + cl = ConsistencyLevel.ONE + peers_query = QueryMessage(query=self._SELECT_PEERS, consistency_level=cl) + local_query = QueryMessage(query=self._SELECT_LOCAL, consistency_level=cl) + peers_result, local_result = connection.wait_for_responses( + peers_query, local_query, timeout=self._timeout) + peers_result = dict_factory(*peers_result.results) partitioner = None @@ -1654,7 +1665,7 @@ class ControlConnection(object): elif event['change_type'] == "UPDATED": self._submit(self.refresh_schema, keyspace, table) - def wait_for_schema_agreement(self, connection=None): + def wait_for_schema_agreement(self, connection=None, preloaded_results=None): # Each schema change typically generates two schema refreshes, one # from the response type and one from the pushed notification. Holding # a lock is just a simple way to cut down on the number of schema queries @@ -1663,10 +1674,18 @@ class ControlConnection(object): if self._is_shutdown: return - log.debug("[control connection] Waiting for schema agreement") if not connection: connection = self._connection + if preloaded_results: + log.debug("[control connection] Attempting to use preloaded results for schema agreement") + + peers_result = preloaded_results[0] + local_result = preloaded_results[1] + if self._do_schemas_match(peers_result, local_result): + return True + + log.debug("[control connection] Waiting for schema agreement") start = self._time.time() elapsed = 0 cl = ConsistencyLevel.ONE @@ -1684,28 +1703,7 @@ class ControlConnection(object): elapsed = self._time.time() - start continue - peers_result = dict_factory(*peers_result.results) - - versions = set() - if local_result.results: - local_row = dict_factory(*local_result.results)[0] - if local_row.get("schema_version"): - versions.add(local_row.get("schema_version")) - - for row in peers_result: - if not row.get("rpc_address") or not row.get("schema_version"): - continue - - rpc = row.get("rpc_address") - if rpc == "0.0.0.0": # TODO ipv6 check - rpc = row.get("peer") - - peer = self._cluster.metadata.get_host(rpc) - if peer and peer.is_up: - versions.add(row.get("schema_version")) - - if len(versions) == 1: - log.debug("[control connection] Schemas match") + if self._do_schemas_match(peers_result, local_result): return True log.debug("[control connection] Schemas mismatched, trying again") @@ -1714,6 +1712,34 @@ class ControlConnection(object): return False + def _do_schemas_match(self, peers_result, local_result): + peers_result = dict_factory(*peers_result.results) + + versions = set() + if local_result.results: + local_row = dict_factory(*local_result.results)[0] + if local_row.get("schema_version"): + versions.add(local_row.get("schema_version")) + + for row in peers_result: + if not row.get("rpc_address") or not row.get("schema_version"): + continue + + rpc = row.get("rpc_address") + if rpc == "0.0.0.0": # TODO ipv6 check + rpc = row.get("peer") + + peer = self._cluster.metadata.get_host(rpc) + if peer and peer.is_up: + versions.add(row.get("schema_version")) + + if len(versions) == 1: + log.debug("[control connection] Schemas match") + return True + + return False + + def _signal_error(self): # try just signaling the cluster, as this will trigger a reconnect # as part of marking the host down diff --git a/tests/unit/test_control_connection.py b/tests/unit/test_control_connection.py index 604034cc..157d0dab 100644 --- a/tests/unit/test_control_connection.py +++ b/tests/unit/test_control_connection.py @@ -90,13 +90,12 @@ class MockConnection(object): [["192.168.1.1", "10.0.0.1", "a", "dc1", "rack1", ["1", "101", "201"]], ["192.168.1.2", "10.0.0.2", "a", "dc1", "rack1", ["2", "102", "202"]]] ] - - def wait_for_responses(self, peer_query, local_query, timeout=None): local_response = ResultMessage( kind=ResultMessage.KIND_ROWS, results=self.local_results) peer_response = ResultMessage( kind=ResultMessage.KIND_ROWS, results=self.peer_results) - return (peer_response, local_response) + + self.wait_for_responses = Mock(return_value=(peer_response, local_response)) class FakeTime(object): @@ -122,6 +121,38 @@ class ControlConnectionTest(unittest.TestCase): self.control_connection._connection = self.connection self.control_connection._time = self.time + def _get_matching_schema_preloaded_results(self): + local_results = [ + ["schema_version", "cluster_name", "data_center", "rack", "partitioner", "tokens"], + [["a", "foocluster", "dc1", "rack1", "Murmur3Partitioner", ["0", "100", "200"]]] + ] + local_response = ResultMessage(kind=ResultMessage.KIND_ROWS, results=local_results) + + peer_results = [ + ["rpc_address", "peer", "schema_version", "data_center", "rack", "tokens"], + [["192.168.1.1", "10.0.0.1", "a", "dc1", "rack1", ["1", "101", "201"]], + ["192.168.1.2", "10.0.0.2", "a", "dc1", "rack1", ["2", "102", "202"]]] + ] + peer_response = ResultMessage(kind=ResultMessage.KIND_ROWS, results=peer_results) + + return (peer_response, local_response) + + def _get_nonmatching_schema_preloaded_results(self): + local_results = [ + ["schema_version", "cluster_name", "data_center", "rack", "partitioner", "tokens"], + [["a", "foocluster", "dc1", "rack1", "Murmur3Partitioner", ["0", "100", "200"]]] + ] + local_response = ResultMessage(kind=ResultMessage.KIND_ROWS, results=local_results) + + peer_results = [ + ["rpc_address", "peer", "schema_version", "data_center", "rack", "tokens"], + [["192.168.1.1", "10.0.0.1", "a", "dc1", "rack1", ["1", "101", "201"]], + ["192.168.1.2", "10.0.0.2", "b", "dc1", "rack1", ["2", "102", "202"]]] + ] + peer_response = ResultMessage(kind=ResultMessage.KIND_ROWS, results=peer_results) + + return (peer_response, local_response) + def test_wait_for_schema_agreement(self): """ Basic test with all schema versions agreeing @@ -130,6 +161,29 @@ class ControlConnectionTest(unittest.TestCase): # the control connection should not have slept at all self.assertEqual(self.time.clock, 0) + def test_wait_for_schema_agreement_uses_preloaded_results_if_given(self): + """ + wait_for_schema_agreement uses preloaded results if given for shared table queries + """ + preloaded_results = self._get_matching_schema_preloaded_results() + + self.assertTrue(self.control_connection.wait_for_schema_agreement(preloaded_results=preloaded_results)) + # the control connection should not have slept at all + self.assertEqual(self.time.clock, 0) + # the connection should not have made any queries if given preloaded results + self.assertEqual(self.connection.wait_for_responses.call_count, 0) + + def test_wait_for_schema_agreement_falls_back_to_querying_if_schemas_dont_match_preloaded_result(self): + """ + wait_for_schema_agreement requery if schema does not match using preloaded results + """ + preloaded_results = self._get_nonmatching_schema_preloaded_results() + + self.assertTrue(self.control_connection.wait_for_schema_agreement(preloaded_results=preloaded_results)) + # the control connection should not have slept at all + self.assertEqual(self.time.clock, 0) + self.assertEqual(self.connection.wait_for_responses.call_count, 1) + def test_wait_for_schema_agreement_fails(self): """ Make sure the control connection sleeps and retries @@ -197,6 +251,32 @@ class ControlConnectionTest(unittest.TestCase): self.assertEqual(host.datacenter, "dc1") self.assertEqual(host.rack, "rack1") + self.assertEqual(self.connection.wait_for_responses.call_count, 1) + + def test_refresh_nodes_and_tokens_uses_preloaded_results_if_given(self): + """ + refresh_nodes_and_tokens uses preloaded results if given for shared table queries + """ + preloaded_results = self._get_matching_schema_preloaded_results() + + self.control_connection._refresh_node_list_and_token_map(self.connection, preloaded_results=preloaded_results) + meta = self.cluster.metadata + self.assertEqual(meta.partitioner, 'Murmur3Partitioner') + self.assertEqual(meta.cluster_name, 'foocluster') + + # check token map + self.assertEqual(sorted(meta.all_hosts()), sorted(meta.token_map.keys())) + for token_list in meta.token_map.values(): + self.assertEqual(3, len(token_list)) + + # check datacenter/rack + for host in meta.all_hosts(): + self.assertEqual(host.datacenter, "dc1") + self.assertEqual(host.rack, "rack1") + + # the connection should not have made any queries if given preloaded results + self.assertEqual(self.connection.wait_for_responses.call_count, 0) + def test_refresh_nodes_and_tokens_no_partitioner(self): """ Test handling of an unknown partitioner. From 4476bda0e782822eedc06122ef5840f2363839a7 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Tue, 8 Apr 2014 11:41:35 -0500 Subject: [PATCH 15/42] Update changelog --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f9db0899..1d37c845 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -37,6 +37,9 @@ Other user-defined type support. (github issue #90) * Better error message when libevwrapper is not found * Only try to import scales when metrics are enabled (github issue #92) +* Cut down on the number of queries executing when a new Cluster + connects and when the control connection has to reconnect (github issue #104, + PYTHON-59) 1.0.2 ===== From 7c6c24001f62abe86dc546343dcd43461bd966b6 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Tue, 8 Apr 2014 11:58:23 -0500 Subject: [PATCH 16/42] Issue warning log when schema versions don't match --- CHANGELOG.rst | 1 + cassandra/cluster.py | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1d37c845..0375fc35 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -40,6 +40,7 @@ Other * Cut down on the number of queries executing when a new Cluster connects and when the control connection has to reconnect (github issue #104, PYTHON-59) +* Issue warning log when schema versions do not match 1.0.2 ===== diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 35d6b92e..7186067b 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -4,6 +4,7 @@ This module houses the main classes you will interact with, """ from __future__ import absolute_import +from collections import defaultdict from concurrent.futures import ThreadPoolExecutor import logging import socket @@ -1682,7 +1683,8 @@ class ControlConnection(object): peers_result = preloaded_results[0] local_result = preloaded_results[1] - if self._do_schemas_match(peers_result, local_result): + schema_mismatches = self._get_schema_mismatches(peers_result, local_result, connection.host) + if schema_mismatches is None: return True log.debug("[control connection] Waiting for schema agreement") @@ -1690,6 +1692,7 @@ class ControlConnection(object): elapsed = 0 cl = ConsistencyLevel.ONE total_timeout = self._cluster.max_schema_agreement_wait + schema_mismatches = None while elapsed < total_timeout: peers_query = QueryMessage(query=self._SELECT_SCHEMA_PEERS, consistency_level=cl) local_query = QueryMessage(query=self._SELECT_SCHEMA_LOCAL, consistency_level=cl) @@ -1703,23 +1706,26 @@ class ControlConnection(object): elapsed = self._time.time() - start continue - if self._do_schemas_match(peers_result, local_result): + schema_mismatches = self._get_schema_mismatches(peers_result, local_result, connection.host) + if schema_mismatches is None: return True log.debug("[control connection] Schemas mismatched, trying again") self._time.sleep(0.2) elapsed = self._time.time() - start + log.warn("Node %s is reporting a schema disagreement: %s", + connection.host, schema_mismatches) return False - def _do_schemas_match(self, peers_result, local_result): + def _get_schema_mismatches(self, peers_result, local_result, local_address): peers_result = dict_factory(*peers_result.results) - versions = set() + versions = defaultdict(set) if local_result.results: local_row = dict_factory(*local_result.results)[0] if local_row.get("schema_version"): - versions.add(local_row.get("schema_version")) + versions[local_row.get("schema_version")].add(local_address) for row in peers_result: if not row.get("rpc_address") or not row.get("schema_version"): @@ -1731,14 +1737,13 @@ class ControlConnection(object): peer = self._cluster.metadata.get_host(rpc) if peer and peer.is_up: - versions.add(row.get("schema_version")) + versions[row.get("schema_version")].add(rpc) if len(versions) == 1: log.debug("[control connection] Schemas match") - return True - - return False + return None + return dict((version, list(nodes)) for version, nodes in versions.iteritems()) def _signal_error(self): # try just signaling the cluster, as this will trigger a reconnect From 69c56bde15a7d6c03fdc91bbc967bf8b6efdfee3 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 3 Apr 2014 14:27:35 -0500 Subject: [PATCH 17/42] Add loadbalancing tests --- .../long/test_loadbalancingpolicies.py | 518 ++++++++++++++++++ 1 file changed, 518 insertions(+) create mode 100644 tests/integration/long/test_loadbalancingpolicies.py diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py new file mode 100644 index 00000000..84b1ba2b --- /dev/null +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -0,0 +1,518 @@ +import struct +from cassandra import ConsistencyLevel, Unavailable +from cassandra.cluster import Cluster +from cassandra.policies import RoundRobinPolicy, DCAwareRoundRobinPolicy, \ + TokenAwarePolicy, WhiteListRoundRobinPolicy +from cassandra.query import SimpleStatement +from tests.integration import use_multidc, use_singledc +from tests.integration.long.utils import wait_for_up, create_schema, \ + CoordinatorStats, force_stop, wait_for_down, decommission, start, ring, \ + bootstrap, stop, IP_FORMAT + +try: + import unittest2 as unittest +except ImportError: + import unittest # noqa + + +class LoadBalancingPolicyTests(unittest.TestCase): + def setUp(self): + self.coordinator_stats = CoordinatorStats() + self.prepared = None + + def teardown(self): + use_singledc() + + def _insert(self, session, keyspace, count=12, + consistency_level=ConsistencyLevel.ONE): + session.execute('USE %s' % keyspace) + for i in range(count): + ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', + consistency_level=consistency_level) + session.execute(ss) + + def _query(self, session, keyspace, count=12, + consistency_level=ConsistencyLevel.ONE, use_prepared=False): + if use_prepared: + query_string = 'SELECT * FROM %s.cf WHERE k = ?' % keyspace + if not self.prepared or self.prepared.query_string != query_string: + self.prepared = session.prepare(query_string) + + for i in range(count): + self.coordinator_stats.add_coordinator(session.execute_async(self.prepared.bind((0,)))) + else: + routing_key = struct.pack('>i', 0) + for i in range(count): + ss = SimpleStatement('SELECT * FROM %s.cf WHERE k = 0' % keyspace, + consistency_level=consistency_level, + routing_key=routing_key) + self.coordinator_stats.add_coordinator(session.execute_async(ss)) + + + def test_roundrobin(self): + use_singledc() + keyspace = 'test_roundrobin' + cluster = Cluster( + load_balancing_policy=RoundRobinPolicy()) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3) + + create_schema(session, keyspace, replication_factor=3) + self._insert(session, keyspace) + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 4) + self.coordinator_stats.assert_query_count_equals(self, 2, 4) + self.coordinator_stats.assert_query_count_equals(self, 3, 4) + + try: + force_stop(3) + wait_for_down(cluster, 3) + + self.coordinator_stats.reset_counts() + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 6) + self.coordinator_stats.assert_query_count_equals(self, 2, 6) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + + decommission(1) + start(3) + wait_for_down(cluster, 1) + wait_for_up(cluster, 3) + + self.coordinator_stats.reset_counts() + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 6) + self.coordinator_stats.assert_query_count_equals(self, 3, 6) + finally: + try: start(1) + except: pass + try: start(3) + except: pass + + wait_for_up(cluster, 3) + + def test_roundrobin_two_dcs(self): + use_multidc([2,2]) + keyspace = 'test_roundrobin_two_dcs' + cluster = Cluster( + load_balancing_policy=RoundRobinPolicy()) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3, wait=False) + wait_for_up(cluster, 4) + + create_schema(session, keyspace, replication_strategy=[2,2]) + self._insert(session, keyspace) + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 3) + self.coordinator_stats.assert_query_count_equals(self, 2, 3) + self.coordinator_stats.assert_query_count_equals(self, 3, 3) + self.coordinator_stats.assert_query_count_equals(self, 4, 3) + + force_stop(1) + bootstrap(5, 'dc3') + + # reset control connection + self._insert(session, keyspace, count=1000) + + wait_for_up(cluster, 5) + + self.coordinator_stats.reset_counts() + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 3) + self.coordinator_stats.assert_query_count_equals(self, 3, 3) + self.coordinator_stats.assert_query_count_equals(self, 4, 3) + self.coordinator_stats.assert_query_count_equals(self, 5, 3) + + def test_roundrobin_two_dcs_2(self): + use_multidc([2,2]) + keyspace = 'test_roundrobin_two_dcs_2' + cluster = Cluster( + load_balancing_policy=RoundRobinPolicy()) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3, wait=False) + wait_for_up(cluster, 4) + + create_schema(session, keyspace, replication_strategy=[2,2]) + self._insert(session, keyspace) + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 3) + self.coordinator_stats.assert_query_count_equals(self, 2, 3) + self.coordinator_stats.assert_query_count_equals(self, 3, 3) + self.coordinator_stats.assert_query_count_equals(self, 4, 3) + + force_stop(1) + bootstrap(5, 'dc1') + + # reset control connection + self._insert(session, keyspace, count=1000) + + wait_for_up(cluster, 5) + + self.coordinator_stats.reset_counts() + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 3) + self.coordinator_stats.assert_query_count_equals(self, 3, 3) + self.coordinator_stats.assert_query_count_equals(self, 4, 3) + self.coordinator_stats.assert_query_count_equals(self, 5, 3) + + def test_dc_aware_roundrobin_two_dcs(self): + use_multidc([3,2]) + keyspace = 'test_dc_aware_roundrobin_two_dcs' + cluster = Cluster( + load_balancing_policy=DCAwareRoundRobinPolicy('dc1')) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3, wait=False) + wait_for_up(cluster, 4, wait=False) + wait_for_up(cluster, 5) + + create_schema(session, keyspace, replication_strategy=[2, 2]) + self._insert(session, keyspace, count=12000) + self._query(session, keyspace, count=12000) + + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 181, in test_dc_aware_roundrobin_two_dcs + # self.coordinator_stats.assert_query_count_equals(self, 1, 4000) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 35, in assert_query_count_equals + # expected, ip, self.coordinator_counts[ip], dict(self.coordinator_counts))) + # AssertionError: Expected 4000 queries to 127.0.0.1, but got 2400. Query counts: {'127.0.0.3': 7200, '127.0.0.2': 2400, '127.0.0.1': 2400} + # -------------------- >> begin captured logging << -------------------- + # self.coordinator_stats.assert_query_count_equals(self, 1, 4000) + # self.coordinator_stats.assert_query_count_equals(self, 2, 4000) + # self.coordinator_stats.assert_query_count_equals(self, 3, 4000) + # self.coordinator_stats.assert_query_count_equals(self, 4, 0) + # self.coordinator_stats.assert_query_count_equals(self, 5, 0) + + def test_dc_aware_roundrobin_two_dcs_2(self): + use_multidc([3,2]) + keyspace = 'test_dc_aware_roundrobin_two_dcs_2' + cluster = Cluster( + load_balancing_policy=DCAwareRoundRobinPolicy('dc2')) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3, wait=False) + wait_for_up(cluster, 4, wait=False) + wait_for_up(cluster, 5) + + create_schema(session, keyspace, replication_strategy=[2, 2]) + self._insert(session, keyspace, count=12000) + self._query(session, keyspace, count=12000) + + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 214, in test_dc_aware_roundrobin_two_dcs_2 + # self.coordinator_stats.assert_query_count_equals(self, 4, 6000) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 35, in assert_query_count_equals + # expected, ip, self.coordinator_counts[ip], dict(self.coordinator_counts))) + # AssertionError: Expected 6000 queries to 127.0.0.4, but got 2400. Query counts: {'127.0.0.5': 9600, '127.0.0.4': 2400, '127.0.0.3': 0, '127.0.0.2': 0, '127.0.0.1': 0} + # -------------------- >> begin captured logging << -------------------- + # self.coordinator_stats.assert_query_count_equals(self, 1, 0) + # self.coordinator_stats.assert_query_count_equals(self, 2, 0) + # self.coordinator_stats.assert_query_count_equals(self, 3, 0) + # self.coordinator_stats.assert_query_count_equals(self, 4, 6000) + # self.coordinator_stats.assert_query_count_equals(self, 5, 6000) + + def test_dc_aware_roundrobin_one_remote_host(self): + use_multidc([2,2]) + keyspace = 'test_dc_aware_roundrobin_one_remote_host_2' + cluster = Cluster( + load_balancing_policy=DCAwareRoundRobinPolicy('dc2', used_hosts_per_remote_dc=1)) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3, wait=False) + wait_for_up(cluster, 4) + + create_schema(session, keyspace, replication_strategy=[2, 2]) + self._insert(session, keyspace, count=12000) + self._query(session, keyspace, count=12000) + + # Differing distributions + # self.coordinator_stats.assert_query_count_equals(self, 1, 0) + # self.coordinator_stats.assert_query_count_equals(self, 2, 0) + # self.coordinator_stats.assert_query_count_equals(self, 3, 6000) + # self.coordinator_stats.assert_query_count_equals(self, 4, 6000) + + self.coordinator_stats.reset_counts() + bootstrap(5, 'dc1') + wait_for_up(cluster, 5) + + self._query(session, keyspace) + + # Differing distributions + # self.coordinator_stats.assert_query_count_equals(self, 1, 0) + # self.coordinator_stats.assert_query_count_equals(self, 2, 0) + # self.coordinator_stats.assert_query_count_equals(self, 3, 6) + # self.coordinator_stats.assert_query_count_equals(self, 4, 6) + # self.coordinator_stats.assert_query_count_equals(self, 5, 0) + + self.coordinator_stats.reset_counts() + decommission(3) + decommission(4) + wait_for_down(cluster, 3, wait=True) + wait_for_down(cluster, 4, wait=True) + + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 242, in test_dc_aware_roundrobin_one_remote_host + # self._query(session, keyspace) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 36, in _query + # self.coordinator_stats.add_coordinator(session.execute_async(ss)) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 21, in add_coordinator + # coordinator = future._current_host.address + # AttributeError: 'NoneType' object has no attribute 'address' + # -------------------- >> begin captured logging << -------------------- + # self._query(session, keyspace) + + # self.coordinator_stats.assert_query_count_equals(self, 1, 0) + # self.coordinator_stats.assert_query_count_equals(self, 2, 0) + # self.coordinator_stats.assert_query_count_equals(self, 3, 0) + # self.coordinator_stats.assert_query_count_equals(self, 4, 0) + # self.coordinator_stats.assert_query_count_equals(self, 5, 12) + + self.coordinator_stats.reset_counts() + decommission(5) + wait_for_down(cluster, 5, wait=True) + + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 266, in test_dc_aware_roundrobin_one_remote_host + # self._query(session, keyspace) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 36, in _query + # self.coordinator_stats.add_coordinator(session.execute_async(ss)) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 21, in add_coordinator + # coordinator = future._current_host.address + # AttributeError: 'NoneType' object has no attribute 'address' + # -------------------- >> begin captured logging << -------------------- + # self._query(session, keyspace) + # + # self.coordinator_stats.assert_query_count_equals(self, 1, 12) + # self.coordinator_stats.assert_query_count_equals(self, 2, 0) + # self.coordinator_stats.assert_query_count_equals(self, 3, 0) + # self.coordinator_stats.assert_query_count_equals(self, 4, 0) + # self.coordinator_stats.assert_query_count_equals(self, 5, 0) + + self.coordinator_stats.reset_counts() + decommission(1) + wait_for_down(cluster, 1, wait=True) + + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 288, in test_dc_aware_roundrobin_one_remote_host + # self._query(session, keyspace) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 36, in _query + # self.coordinator_stats.add_coordinator(session.execute_async(ss)) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 21, in add_coordinator + # coordinator = future._current_host.address + # AttributeError: 'NoneType' object has no attribute 'address' + # -------------------- >> begin captured logging << -------------------- + # self._query(session, keyspace) + # + # self.coordinator_stats.assert_query_count_equals(self, 1, 0) + # self.coordinator_stats.assert_query_count_equals(self, 2, 12) + # self.coordinator_stats.assert_query_count_equals(self, 3, 0) + # self.coordinator_stats.assert_query_count_equals(self, 4, 0) + # self.coordinator_stats.assert_query_count_equals(self, 5, 0) + + self.coordinator_stats.reset_counts() + force_stop(2) + + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 309, in test_dc_aware_roundrobin_one_remote_host + # self._query(session, keyspace) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 36, in _query + # self.coordinator_stats.add_coordinator(session.execute_async(ss)) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 21, in add_coordinator + # coordinator = future._current_host.address + # AttributeError: 'NoneType' object has no attribute 'address' + # -------------------- >> begin captured logging << -------------------- + # should throw an error about not being able to connect + # self._query(session, keyspace) + + def test_token_aware(self): + keyspace = 'test_token_aware' + self.token_aware(keyspace) + + def test_token_aware_prepared(self): + keyspace = 'test_token_aware_prepared' + self.token_aware(keyspace, True) + + def token_aware(self, keyspace, use_prepared=False): + use_singledc() + cluster = Cluster( + load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy())) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3) + + create_schema(session, keyspace, replication_factor=1) + self._insert(session, keyspace) + self._query(session, keyspace, use_prepared=use_prepared) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 12) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + + self.coordinator_stats.reset_counts() + self._query(session, keyspace, use_prepared=use_prepared) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 12) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + + self.coordinator_stats.reset_counts() + force_stop(2) + wait_for_down(cluster, 2, wait=True) + + try: + self._query(session, keyspace, use_prepared=use_prepared) + self.fail() + except Unavailable as e: + self.assertEqual(e.consistency, 1) + self.assertEqual(e.required_replicas, 1) + self.assertEqual(e.alive_replicas, 0) + + self.coordinator_stats.reset_counts() + start(2) + wait_for_up(cluster, 2, wait=True) + + self._query(session, keyspace, use_prepared=use_prepared) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 12) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + + self.coordinator_stats.reset_counts() + decommission(2) + wait_for_down(cluster, 2, wait=True) + + self._query(session, keyspace, use_prepared=use_prepared) + + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 328, in test_token_aware + # self.token_aware() + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 387, in token_aware + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 35, in assert_query_count_equals + # expected, ip, self.coordinator_counts[ip], dict(self.coordinator_counts))) + # AssertionError: Expected 12 queries to 127.0.0.1, but got 6. Query counts: {'127.0.0.3': 6, '127.0.0.1': 6} + # -------------------- >> begin captured logging << -------------------- + # This is different than the java-driver, but I just wanted to make sure it's intended + # self.coordinator_stats.assert_query_count_equals(self, 1, 12) + # self.coordinator_stats.assert_query_count_equals(self, 2, 0) + # self.coordinator_stats.assert_query_count_equals(self, 3, 0) + + def test_token_aware_composite_key(self): + use_singledc() + keyspace = 'test_token_aware_composite_key' + table = 'composite' + cluster = Cluster( + load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy())) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3) + + create_schema(session, keyspace, replication_factor=2) + session.execute('CREATE TABLE %s (' + 'k1 int, ' + 'k2 int, ' + 'i int, ' + 'PRIMARY KEY ((k1, k2)))' % table) + + prepared = session.prepare('INSERT INTO %s ' + '(k1, k2, i) ' + 'VALUES ' + '(?, ?, ?)' % table) + session.execute(prepared.bind((1, 2, 3))) + + results = session.execute('SELECT * FROM %s WHERE k1 = 1 AND k2 = 2' % table) + self.assertTrue(len(results) == 1) + self.assertTrue(results[0].i) + + def test_token_aware_with_rf_2(self, use_prepared=False): + use_singledc() + keyspace = 'test_token_aware_with_rf_2' + cluster = Cluster( + load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy())) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3) + + create_schema(session, keyspace, replication_factor=2) + self._insert(session, keyspace) + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 12) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + + self.coordinator_stats.reset_counts() + stop(2) + wait_for_down(cluster, 2, wait=True) + + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 0) + self.coordinator_stats.assert_query_count_equals(self, 3, 12) + + def test_white_list(self): + use_singledc() + keyspace = 'test_white_list' + + # BUG? Can only connect to the cluster via nodes in the whitelist? + # Perhaps we should be more lenient here? + cluster = Cluster(('127.0.0.2',), + load_balancing_policy=WhiteListRoundRobinPolicy((IP_FORMAT % 2,))) + session = cluster.connect() + wait_for_up(cluster, 1, wait=False) + wait_for_up(cluster, 2, wait=False) + wait_for_up(cluster, 3) + + create_schema(session, keyspace) + self._insert(session, keyspace) + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 12) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + + self.coordinator_stats.reset_counts() + decommission(2) + wait_for_down(cluster, 2, wait=True) + + # I was expecting another type of error since only IP_FORMAT % 2 is whitelisted + # ---------------------------------------------------------------------- + # Traceback (most recent call last): + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 506, in test_white_list + # self._query(session, keyspace) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 49, in _query + # self.coordinator_stats.add_coordinator(session.execute_async(ss)) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 22, in add_coordinator + # coordinator = future._current_host.address + # AttributeError: 'NoneType' object has no attribute 'address' + # -------------------- >> begin captured logging << -------------------- + # self._query(session, keyspace) From 910d357beb99e2d8fdfda990a24a738ab2353b77 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 3 Apr 2014 14:32:39 -0500 Subject: [PATCH 18/42] Multiple cluster support Conflicts: tests/integration/__init__.py --- tests/integration/__init__.py | 56 ++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 7f7f066f..b8a9e9f3 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,3 +1,6 @@ +from ccmlib.common import current_cluster_name +import time + try: import unittest2 as unittest except ImportError: @@ -17,6 +20,7 @@ except ImportError as e: raise unittest.SkipTest('ccm is a dependency for integration tests:', e) CLUSTER_NAME = 'test_cluster' +MULTIDC_CLUSTER_NAME = 'multidc_test_cluster' CCM_CLUSTER = None DEFAULT_CASSANDRA_VERSION = '1.2.15' @@ -84,8 +88,44 @@ def setup_package(): setup_test_keyspace() +def use_multidc(dc_list): + teardown_package() + try: + try: + cluster = CCMCluster.load(path, MULTIDC_CLUSTER_NAME) + log.debug("Found existing ccm test multi-dc cluster, clearing") + cluster.clear() + except Exception: + log.debug("Creating new ccm test multi-dc cluster") + cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION) + cluster.set_configuration_options({'start_native_transport': True}) + common.switch_cluster(path, MULTIDC_CLUSTER_NAME) + cluster.populate(dc_list) + + log.debug("Starting ccm test cluster") + cluster.start(wait_for_binary_proto=True) + except Exception: + log.exception("Failed to start ccm cluster:") + raise + + global CCM_CLUSTER + CCM_CLUSTER = cluster + setup_test_keyspace() + log.debug("Switched to multidc cluster") + + +def use_singledc(): + teardown_package() + + setup_package() + log.debug("Switched to singledc cluster") + + def setup_test_keyspace(): - cluster = Cluster() + # wait for nodes to startup + time.sleep(10) + + cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = cluster.connect() try: @@ -120,11 +160,19 @@ def setup_test_keyspace(): def teardown_package(): - if CCM_CLUSTER: + for cluster_name in [CLUSTER_NAME, MULTIDC_CLUSTER_NAME]: try: - CCM_CLUSTER.clear() + cluster = CCMCluster.load(path, cluster_name) + + try: + cluster.clear() + cluster.remove() + log.info('Cleared cluster: %s' % cluster_name) + except Exception: + log.exception('Failed to clear cluster: %s' % cluster_name) + except Exception: - log.exception("Failed to clear cluster") + log.warn('Did not find cluster: %s' % cluster_name) class UpDownWaiter(object): From c8431103420cb2ea9392b9848014fc8005e86b6c Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 3 Apr 2014 14:33:18 -0500 Subject: [PATCH 19/42] decommission and bootstrap ccm commands --- tests/integration/long/utils.py | 36 +++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index a83d255b..f83158c7 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -2,11 +2,13 @@ import logging import time from collections import defaultdict +from ccmlib.node import Node from cassandra.decoder import named_tuple_factory -from tests.integration import get_node +from tests.integration import get_node, get_cluster +IP_FORMAT = '127.0.0.%s' log = logging.getLogger(__name__) @@ -78,6 +80,32 @@ def force_stop(node): log.debug("Node %s was stopped", node) +def decommission(node): + get_node(node).decommission() + get_node(node).stop() + + +def bootstrap(node, data_center=None, token=None): + node_instance = Node('node%s' % node, + get_cluster(), + auto_bootstrap=False, + thrift_interface=(IP_FORMAT % node, 9160), + storage_interface=(IP_FORMAT % node, 7000), + jmx_port=str(7000 + 100 * node), + remote_debug_port=0, + initial_token=token if token else node*10) + get_cluster().add(node_instance, is_seed=False, data_center=data_center) + + try: + start(node) + except: + # Try only twice + try: + start(node) + except: + log.error('Added node failed to start twice.') + + def ring(node): print 'From node%s:' % node get_node(node).nodetool('ring') @@ -85,19 +113,19 @@ def ring(node): def wait_for_up(cluster, node, wait=True): while True: - host = cluster.metadata.get_host('127.0.0.%s' % node) + host = cluster.metadata.get_host(IP_FORMAT % node) time.sleep(0.1) if host and host.is_up: # BUG: shouldn't have to, but we do if wait: - time.sleep(5) + time.sleep(20) return def wait_for_down(cluster, node, wait=True): log.debug("Waiting for node %s to be down", node) while True: - host = cluster.metadata.get_host('127.0.0.%s' % node) + host = cluster.metadata.get_host(IP_FORMAT % node) time.sleep(0.1) if not host or not host.is_up: # BUG: shouldn't have to, but we do From 1586ce473a6d8739bf298f14fa7e63ec8bcc3303 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 3 Apr 2014 14:33:34 -0500 Subject: [PATCH 20/42] Spotted small bug in random test --- tests/integration/standard/test_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index c0f8670a..fcfa9a37 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -400,7 +400,7 @@ class TestCodeCoverage(unittest.TestCase): get_replicas = cluster.metadata.token_map.get_replicas for ksname in ('test1rf', 'test2rf', 'test3rf'): - self.assertNotEqual(list(get_replicas('test3rf', ring[0])), []) + self.assertNotEqual(list(get_replicas(ksname, ring[0])), []) for i, token in enumerate(ring): self.assertEqual(set(get_replicas('test3rf', token)), set(owners)) From ad7a7dd3e17295ac856408383214e2fc4a056e7d Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 3 Apr 2014 14:59:15 -0500 Subject: [PATCH 21/42] Remove try/except blocks and use use_*dc() instead --- .../long/test_loadbalancingpolicies.py | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 84b1ba2b..905b34e1 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -67,35 +67,27 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats.assert_query_count_equals(self, 2, 4) self.coordinator_stats.assert_query_count_equals(self, 3, 4) - try: - force_stop(3) - wait_for_down(cluster, 3) + force_stop(3) + wait_for_down(cluster, 3) - self.coordinator_stats.reset_counts() - self._query(session, keyspace) + self.coordinator_stats.reset_counts() + self._query(session, keyspace) - self.coordinator_stats.assert_query_count_equals(self, 1, 6) - self.coordinator_stats.assert_query_count_equals(self, 2, 6) - self.coordinator_stats.assert_query_count_equals(self, 3, 0) + self.coordinator_stats.assert_query_count_equals(self, 1, 6) + self.coordinator_stats.assert_query_count_equals(self, 2, 6) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) - decommission(1) - start(3) - wait_for_down(cluster, 1) - wait_for_up(cluster, 3) + decommission(1) + start(3) + wait_for_down(cluster, 1) + wait_for_up(cluster, 3) - self.coordinator_stats.reset_counts() - self._query(session, keyspace) + self.coordinator_stats.reset_counts() + self._query(session, keyspace) - self.coordinator_stats.assert_query_count_equals(self, 1, 0) - self.coordinator_stats.assert_query_count_equals(self, 2, 6) - self.coordinator_stats.assert_query_count_equals(self, 3, 6) - finally: - try: start(1) - except: pass - try: start(3) - except: pass - - wait_for_up(cluster, 3) + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 6) + self.coordinator_stats.assert_query_count_equals(self, 3, 6) def test_roundrobin_two_dcs(self): use_multidc([2,2]) From 7ef98cc73e61020cdb36362bb348dc818ba368b0 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 3 Apr 2014 18:56:35 -0500 Subject: [PATCH 22/42] Increase wait_for_down. Replace log messages. --- tests/integration/long/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index f83158c7..a1f46f22 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -118,7 +118,9 @@ def wait_for_up(cluster, node, wait=True): if host and host.is_up: # BUG: shouldn't have to, but we do if wait: + log.debug("Sleeping 5s until host is up") time.sleep(20) + log.debug("Done waiting for node %s to be up", node) return @@ -130,7 +132,7 @@ def wait_for_down(cluster, node, wait=True): if not host or not host.is_up: # BUG: shouldn't have to, but we do if wait: - log.debug("Sleeping 5s until host is up") - time.sleep(5) + log.debug("Sleeping 5s until host is down") + time.sleep(10) log.debug("Done waiting for node %s to be down", node) return From d2306e5d58fd9d24e8f208a4dbfe45780fa46f17 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Tue, 8 Apr 2014 18:50:13 -0500 Subject: [PATCH 23/42] Grab futures before counting coordinators --- tests/integration/__init__.py | 2 +- tests/integration/long/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index b8a9e9f3..d7f2da83 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -124,7 +124,7 @@ def use_singledc(): def setup_test_keyspace(): # wait for nodes to startup time.sleep(10) - + cluster = Cluster(protocol_version=PROTOCOL_VERSION) session = cluster.connect() diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index a1f46f22..25903cdb 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -19,6 +19,7 @@ class CoordinatorStats(): self.coordinator_counts = defaultdict(int) def add_coordinator(self, future): + future.result() coordinator = future._current_host.address self.coordinator_counts[coordinator] += 1 From 648609133dff8352123e5faa149d290740017b4a Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Tue, 8 Apr 2014 19:42:54 -0500 Subject: [PATCH 24/42] Remove double future.result() --- tests/integration/long/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index 25903cdb..99259486 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -25,7 +25,6 @@ class CoordinatorStats(): if future._errors: log.error('future._errors: %s', future._errors) - future.result() def reset_counts(self): self.coordinator_counts = defaultdict(int) From efeb93a2fcd9192fbfae0dfdc626c37c8f1c680b Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Tue, 8 Apr 2014 20:19:26 -0500 Subject: [PATCH 25/42] Confirm stacktrace --- tests/integration/__init__.py | 4 ++-- tests/integration/long/test_loadbalancingpolicies.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index d7f2da83..1d67fcf9 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -97,7 +97,7 @@ def use_multidc(dc_list): cluster.clear() except Exception: log.debug("Creating new ccm test multi-dc cluster") - cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_version=CASSANDRA_VERSION) + cluster = CCMCluster(path, MULTIDC_CLUSTER_NAME, cassandra_version=DEFAULT_CASSANDRA_VERSION) cluster.set_configuration_options({'start_native_transport': True}) common.switch_cluster(path, MULTIDC_CLUSTER_NAME) cluster.populate(dc_list) @@ -125,7 +125,7 @@ def setup_test_keyspace(): # wait for nodes to startup time.sleep(10) - cluster = Cluster(protocol_version=PROTOCOL_VERSION) + cluster = Cluster() session = cluster.connect() try: diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 905b34e1..2c90efaf 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -403,10 +403,11 @@ class LoadBalancingPolicyTests(unittest.TestCase): # ---------------------------------------------------------------------- # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 328, in test_token_aware - # self.token_aware() - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 387, in token_aware - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 35, in assert_query_count_equals + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 346, in test_token_aware + # self.token_aware(keyspace) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 414, in token_aware + # self.coordinator_stats.assert_query_count_equals(self, 1, 12) + # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 36, in assert_query_count_equals # expected, ip, self.coordinator_counts[ip], dict(self.coordinator_counts))) # AssertionError: Expected 12 queries to 127.0.0.1, but got 6. Query counts: {'127.0.0.3': 6, '127.0.0.1': 6} # -------------------- >> begin captured logging << -------------------- From c711ccacac4babfda0b1c17b3a42f0949c36c184 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Wed, 9 Apr 2014 13:55:25 -0500 Subject: [PATCH 26/42] Fix DCAware load imbalance due to set orderings --- CHANGELOG.rst | 2 ++ cassandra/policies.py | 26 +++++++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0375fc35..22b2745f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -30,6 +30,8 @@ Bug Fixes and rack information has been set, if possible. * Avoid KeyError when updating metadata after droping a table (github issues #97, #98) +* Use tuples instead of sets for DCAwareLoadBalancingPolicy to ensure equal + distribution of requests Other ----- diff --git a/cassandra/policies.py b/cassandra/policies.py index a563d7f2..1d645781 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -210,7 +210,7 @@ class DCAwareRoundRobinPolicy(LoadBalancingPolicy): def populate(self, cluster, hosts): for dc, dc_hosts in groupby(hosts, lambda h: self._dc(h)): - self._dc_live_hosts[dc] = frozenset(dc_hosts) + self._dc_live_hosts[dc] = tuple(set(dc_hosts)) # position is currently only used for local hosts local_live = self._dc_live_hosts.get(self.local_dc) @@ -244,7 +244,7 @@ class DCAwareRoundRobinPolicy(LoadBalancingPolicy): pos = self._position self._position += 1 - local_live = list(self._dc_live_hosts.get(self.local_dc, ())) + local_live = self._dc_live_hosts.get(self.local_dc, ()) pos = (pos % len(local_live)) if local_live else 0 for host in islice(cycle(local_live), pos, pos + len(local_live)): yield host @@ -253,32 +253,36 @@ class DCAwareRoundRobinPolicy(LoadBalancingPolicy): if dc == self.local_dc: continue - for host in list(current_dc_hosts)[:self.used_hosts_per_remote_dc]: + for host in current_dc_hosts[:self.used_hosts_per_remote_dc]: yield host def on_up(self, host): dc = self._dc(host) with self._hosts_lock: - current_hosts = self._dc_live_hosts.setdefault(dc, frozenset()) - self._dc_live_hosts[dc] = current_hosts.union((host, )) + current_hosts = self._dc_live_hosts.setdefault(dc, ()) + if host not in current_hosts: + self._dc_live_hosts[dc] = current_hosts + (host, ) def on_down(self, host): dc = self._dc(host) with self._hosts_lock: - current_hosts = self._dc_live_hosts.setdefault(dc, frozenset()) - self._dc_live_hosts[dc] = current_hosts.difference((host, )) + current_hosts = self._dc_live_hosts.setdefault(dc, ()) + if host in current_hosts: + self._dc_live_hosts[dc] = tuple(h for h in current_hosts if h != host) def on_add(self, host): dc = self._dc(host) with self._hosts_lock: - current_hosts = self._dc_live_hosts.setdefault(dc, frozenset()) - self._dc_live_hosts[dc] = current_hosts.union((host, )) + current_hosts = self._dc_live_hosts.setdefault(dc, ()) + if host not in current_hosts: + self._dc_live_hosts[dc] = current_hosts + (host, ) def on_remove(self, host): dc = self._dc(host) with self._hosts_lock: - current_hosts = self._dc_live_hosts.setdefault(dc, frozenset()) - self._dc_live_hosts[dc] = current_hosts.difference((host, )) + current_hosts = self._dc_live_hosts.setdefault(dc, ()) + if host in current_hosts: + self._dc_live_hosts[dc] = tuple(h for h in current_hosts if h != host) class TokenAwarePolicy(LoadBalancingPolicy): From 89b465d40e47427bba296f1e56053f8e52eb56ff Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Wed, 9 Apr 2014 13:56:53 -0500 Subject: [PATCH 27/42] Add *.egg to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b8ce3535..7595803e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.swp *.swo *.so +*.egg *.egg-info .tox .idea/ From e955fb53fcc298e4435d61075c1c5d387eeddf00 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Wed, 9 Apr 2014 14:55:38 -0500 Subject: [PATCH 28/42] Use execute_concurrent --- tests/integration/long/test_loadbalancingpolicies.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 2c90efaf..6acb8c1c 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -1,6 +1,7 @@ import struct from cassandra import ConsistencyLevel, Unavailable from cassandra.cluster import Cluster +from cassandra.concurrent import execute_concurrent_with_args from cassandra.policies import RoundRobinPolicy, DCAwareRoundRobinPolicy, \ TokenAwarePolicy, WhiteListRoundRobinPolicy from cassandra.query import SimpleStatement @@ -29,7 +30,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): for i in range(count): ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', consistency_level=consistency_level) - session.execute(ss) + execute_concurrent_with_args(session, ss, [None] * count) def _query(self, session, keyspace, count=12, consistency_level=ConsistencyLevel.ONE, use_prepared=False): From fd9aafa0d5e4918b25880fda744dfe6721196376 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Wed, 9 Apr 2014 18:38:49 -0500 Subject: [PATCH 29/42] Don't log stacktrace for NoConnectionsAvailable --- cassandra/cluster.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 7186067b..3255f2ce 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -38,7 +38,7 @@ from cassandra.policies import (RoundRobinPolicy, SimpleConvictionPolicy, ExponentialReconnectionPolicy, HostDistance, RetryPolicy) from cassandra.pool import (_ReconnectionHandler, _HostReconnectionHandler, - HostConnectionPool) + HostConnectionPool, NoConnectionsAvailable) from cassandra.query import (SimpleStatement, PreparedStatement, BoundStatement, bind_params, QueryTrace, Statement) @@ -1953,6 +1953,10 @@ class ResponseFuture(object): # TODO get connectTimeout from cluster settings connection = pool.borrow_connection(timeout=2.0) request_id = connection.send_msg(message, cb=cb) + except NoConnectionsAvailable as exc: + log.debug("All connections for host %s are at capacity, moving to the next host", host) + self._errors[host] = exc + return None except Exception as exc: log.debug("Error querying host %s", host, exc_info=True) self._errors[host] = exc From f3d5a5a5aeeffeacb53081e00e3919bb2740909f Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Thu, 10 Apr 2014 19:12:29 -0500 Subject: [PATCH 30/42] Cleaned up and re-tested test cases --- .../long/test_loadbalancingpolicies.py | 200 ++++++------------ tests/integration/long/utils.py | 10 +- 2 files changed, 71 insertions(+), 139 deletions(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 6acb8c1c..08b1a155 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -1,6 +1,6 @@ import struct from cassandra import ConsistencyLevel, Unavailable -from cassandra.cluster import Cluster +from cassandra.cluster import Cluster, NoHostAvailable from cassandra.concurrent import execute_concurrent_with_args from cassandra.policies import RoundRobinPolicy, DCAwareRoundRobinPolicy, \ TokenAwarePolicy, WhiteListRoundRobinPolicy @@ -30,7 +30,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): for i in range(count): ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', consistency_level=consistency_level) - execute_concurrent_with_args(session, ss, [None] * count) + session.execute(ss) def _query(self, session, keyspace, count=12, consistency_level=ConsistencyLevel.ONE, use_prepared=False): @@ -177,22 +177,14 @@ class LoadBalancingPolicyTests(unittest.TestCase): wait_for_up(cluster, 5) create_schema(session, keyspace, replication_strategy=[2, 2]) - self._insert(session, keyspace, count=12000) - self._query(session, keyspace, count=12000) + self._insert(session, keyspace) + self._query(session, keyspace) - # ---------------------------------------------------------------------- - # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 181, in test_dc_aware_roundrobin_two_dcs - # self.coordinator_stats.assert_query_count_equals(self, 1, 4000) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 35, in assert_query_count_equals - # expected, ip, self.coordinator_counts[ip], dict(self.coordinator_counts))) - # AssertionError: Expected 4000 queries to 127.0.0.1, but got 2400. Query counts: {'127.0.0.3': 7200, '127.0.0.2': 2400, '127.0.0.1': 2400} - # -------------------- >> begin captured logging << -------------------- - # self.coordinator_stats.assert_query_count_equals(self, 1, 4000) - # self.coordinator_stats.assert_query_count_equals(self, 2, 4000) - # self.coordinator_stats.assert_query_count_equals(self, 3, 4000) - # self.coordinator_stats.assert_query_count_equals(self, 4, 0) - # self.coordinator_stats.assert_query_count_equals(self, 5, 0) + self.coordinator_stats.assert_query_count_equals(self, 1, 4) + self.coordinator_stats.assert_query_count_equals(self, 2, 4) + self.coordinator_stats.assert_query_count_equals(self, 3, 4) + self.coordinator_stats.assert_query_count_equals(self, 4, 0) + self.coordinator_stats.assert_query_count_equals(self, 5, 0) def test_dc_aware_roundrobin_two_dcs_2(self): use_multidc([3,2]) @@ -207,26 +199,18 @@ class LoadBalancingPolicyTests(unittest.TestCase): wait_for_up(cluster, 5) create_schema(session, keyspace, replication_strategy=[2, 2]) - self._insert(session, keyspace, count=12000) - self._query(session, keyspace, count=12000) + self._insert(session, keyspace) + self._query(session, keyspace) - # ---------------------------------------------------------------------- - # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 214, in test_dc_aware_roundrobin_two_dcs_2 - # self.coordinator_stats.assert_query_count_equals(self, 4, 6000) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 35, in assert_query_count_equals - # expected, ip, self.coordinator_counts[ip], dict(self.coordinator_counts))) - # AssertionError: Expected 6000 queries to 127.0.0.4, but got 2400. Query counts: {'127.0.0.5': 9600, '127.0.0.4': 2400, '127.0.0.3': 0, '127.0.0.2': 0, '127.0.0.1': 0} - # -------------------- >> begin captured logging << -------------------- - # self.coordinator_stats.assert_query_count_equals(self, 1, 0) - # self.coordinator_stats.assert_query_count_equals(self, 2, 0) - # self.coordinator_stats.assert_query_count_equals(self, 3, 0) - # self.coordinator_stats.assert_query_count_equals(self, 4, 6000) - # self.coordinator_stats.assert_query_count_equals(self, 5, 6000) + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 0) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + self.coordinator_stats.assert_query_count_equals(self, 4, 6) + self.coordinator_stats.assert_query_count_equals(self, 5, 6) def test_dc_aware_roundrobin_one_remote_host(self): use_multidc([2,2]) - keyspace = 'test_dc_aware_roundrobin_one_remote_host_2' + keyspace = 'test_dc_aware_roundrobin_one_remote_host' cluster = Cluster( load_balancing_policy=DCAwareRoundRobinPolicy('dc2', used_hosts_per_remote_dc=1)) session = cluster.connect() @@ -236,14 +220,13 @@ class LoadBalancingPolicyTests(unittest.TestCase): wait_for_up(cluster, 4) create_schema(session, keyspace, replication_strategy=[2, 2]) - self._insert(session, keyspace, count=12000) - self._query(session, keyspace, count=12000) + self._insert(session, keyspace) + self._query(session, keyspace) - # Differing distributions - # self.coordinator_stats.assert_query_count_equals(self, 1, 0) - # self.coordinator_stats.assert_query_count_equals(self, 2, 0) - # self.coordinator_stats.assert_query_count_equals(self, 3, 6000) - # self.coordinator_stats.assert_query_count_equals(self, 4, 6000) + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 0) + self.coordinator_stats.assert_query_count_equals(self, 3, 6) + self.coordinator_stats.assert_query_count_equals(self, 4, 6) self.coordinator_stats.reset_counts() bootstrap(5, 'dc1') @@ -251,12 +234,11 @@ class LoadBalancingPolicyTests(unittest.TestCase): self._query(session, keyspace) - # Differing distributions - # self.coordinator_stats.assert_query_count_equals(self, 1, 0) - # self.coordinator_stats.assert_query_count_equals(self, 2, 0) - # self.coordinator_stats.assert_query_count_equals(self, 3, 6) - # self.coordinator_stats.assert_query_count_equals(self, 4, 6) - # self.coordinator_stats.assert_query_count_equals(self, 5, 0) + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 0) + self.coordinator_stats.assert_query_count_equals(self, 3, 6) + self.coordinator_stats.assert_query_count_equals(self, 4, 6) + self.coordinator_stats.assert_query_count_equals(self, 5, 0) self.coordinator_stats.reset_counts() decommission(3) @@ -264,83 +246,49 @@ class LoadBalancingPolicyTests(unittest.TestCase): wait_for_down(cluster, 3, wait=True) wait_for_down(cluster, 4, wait=True) - # ---------------------------------------------------------------------- - # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 242, in test_dc_aware_roundrobin_one_remote_host - # self._query(session, keyspace) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 36, in _query - # self.coordinator_stats.add_coordinator(session.execute_async(ss)) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 21, in add_coordinator - # coordinator = future._current_host.address - # AttributeError: 'NoneType' object has no attribute 'address' - # -------------------- >> begin captured logging << -------------------- - # self._query(session, keyspace) + self._query(session, keyspace) - # self.coordinator_stats.assert_query_count_equals(self, 1, 0) - # self.coordinator_stats.assert_query_count_equals(self, 2, 0) - # self.coordinator_stats.assert_query_count_equals(self, 3, 0) - # self.coordinator_stats.assert_query_count_equals(self, 4, 0) - # self.coordinator_stats.assert_query_count_equals(self, 5, 12) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + self.coordinator_stats.assert_query_count_equals(self, 4, 0) + responses = set() + for node in [1,2,5]: + responses.add(self.coordinator_stats.get_query_count(node)) + self.assertEqual(set([0,0,12]), responses) self.coordinator_stats.reset_counts() decommission(5) wait_for_down(cluster, 5, wait=True) - # ---------------------------------------------------------------------- - # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 266, in test_dc_aware_roundrobin_one_remote_host - # self._query(session, keyspace) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 36, in _query - # self.coordinator_stats.add_coordinator(session.execute_async(ss)) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 21, in add_coordinator - # coordinator = future._current_host.address - # AttributeError: 'NoneType' object has no attribute 'address' - # -------------------- >> begin captured logging << -------------------- - # self._query(session, keyspace) - # - # self.coordinator_stats.assert_query_count_equals(self, 1, 12) - # self.coordinator_stats.assert_query_count_equals(self, 2, 0) - # self.coordinator_stats.assert_query_count_equals(self, 3, 0) - # self.coordinator_stats.assert_query_count_equals(self, 4, 0) - # self.coordinator_stats.assert_query_count_equals(self, 5, 0) + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + self.coordinator_stats.assert_query_count_equals(self, 4, 0) + self.coordinator_stats.assert_query_count_equals(self, 5, 0) + responses = set() + for node in [1,2]: + responses.add(self.coordinator_stats.get_query_count(node)) + self.assertEqual(set([0,12]), responses) self.coordinator_stats.reset_counts() decommission(1) wait_for_down(cluster, 1, wait=True) - # ---------------------------------------------------------------------- - # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 288, in test_dc_aware_roundrobin_one_remote_host - # self._query(session, keyspace) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 36, in _query - # self.coordinator_stats.add_coordinator(session.execute_async(ss)) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 21, in add_coordinator - # coordinator = future._current_host.address - # AttributeError: 'NoneType' object has no attribute 'address' - # -------------------- >> begin captured logging << -------------------- - # self._query(session, keyspace) - # - # self.coordinator_stats.assert_query_count_equals(self, 1, 0) - # self.coordinator_stats.assert_query_count_equals(self, 2, 12) - # self.coordinator_stats.assert_query_count_equals(self, 3, 0) - # self.coordinator_stats.assert_query_count_equals(self, 4, 0) - # self.coordinator_stats.assert_query_count_equals(self, 5, 0) + self._query(session, keyspace) + + self.coordinator_stats.assert_query_count_equals(self, 1, 0) + self.coordinator_stats.assert_query_count_equals(self, 2, 12) + self.coordinator_stats.assert_query_count_equals(self, 3, 0) + self.coordinator_stats.assert_query_count_equals(self, 4, 0) + self.coordinator_stats.assert_query_count_equals(self, 5, 0) self.coordinator_stats.reset_counts() force_stop(2) - # ---------------------------------------------------------------------- - # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 309, in test_dc_aware_roundrobin_one_remote_host - # self._query(session, keyspace) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 36, in _query - # self.coordinator_stats.add_coordinator(session.execute_async(ss)) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 21, in add_coordinator - # coordinator = future._current_host.address - # AttributeError: 'NoneType' object has no attribute 'address' - # -------------------- >> begin captured logging << -------------------- - # should throw an error about not being able to connect - # self._query(session, keyspace) + try: + self._query(session, keyspace) + self.fail() + except NoHostAvailable: + pass def test_token_aware(self): keyspace = 'test_token_aware' @@ -402,20 +350,9 @@ class LoadBalancingPolicyTests(unittest.TestCase): self._query(session, keyspace, use_prepared=use_prepared) - # ---------------------------------------------------------------------- - # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 346, in test_token_aware - # self.token_aware(keyspace) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 414, in token_aware - # self.coordinator_stats.assert_query_count_equals(self, 1, 12) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 36, in assert_query_count_equals - # expected, ip, self.coordinator_counts[ip], dict(self.coordinator_counts))) - # AssertionError: Expected 12 queries to 127.0.0.1, but got 6. Query counts: {'127.0.0.3': 6, '127.0.0.1': 6} - # -------------------- >> begin captured logging << -------------------- - # This is different than the java-driver, but I just wanted to make sure it's intended - # self.coordinator_stats.assert_query_count_equals(self, 1, 12) - # self.coordinator_stats.assert_query_count_equals(self, 2, 0) - # self.coordinator_stats.assert_query_count_equals(self, 3, 0) + self.coordinator_stats.assert_query_count_equals(self, 1, 6) + self.coordinator_stats.assert_query_count_equals(self, 2, 0) + self.coordinator_stats.assert_query_count_equals(self, 3, 6) def test_token_aware_composite_key(self): use_singledc() @@ -477,8 +414,6 @@ class LoadBalancingPolicyTests(unittest.TestCase): use_singledc() keyspace = 'test_white_list' - # BUG? Can only connect to the cluster via nodes in the whitelist? - # Perhaps we should be more lenient here? cluster = Cluster(('127.0.0.2',), load_balancing_policy=WhiteListRoundRobinPolicy((IP_FORMAT % 2,))) session = cluster.connect() @@ -498,15 +433,8 @@ class LoadBalancingPolicyTests(unittest.TestCase): decommission(2) wait_for_down(cluster, 2, wait=True) - # I was expecting another type of error since only IP_FORMAT % 2 is whitelisted - # ---------------------------------------------------------------------- - # Traceback (most recent call last): - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 506, in test_white_list - # self._query(session, keyspace) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/test_loadbalancingpolicies.py", line 49, in _query - # self.coordinator_stats.add_coordinator(session.execute_async(ss)) - # File "/Users/joaquin/repos/python-driver/tests/integration/long/utils.py", line 22, in add_coordinator - # coordinator = future._current_host.address - # AttributeError: 'NoneType' object has no attribute 'address' - # -------------------- >> begin captured logging << -------------------- - # self._query(session, keyspace) + try: + self._query(session, keyspace) + self.fail() + except NoHostAvailable: + pass diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index 99259486..470b68a0 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -29,9 +29,13 @@ class CoordinatorStats(): def reset_counts(self): self.coordinator_counts = defaultdict(int) + def get_query_count(self, node): + ip = '127.0.0.%d' % node + return self.coordinator_counts[ip] + def assert_query_count_equals(self, testcase, node, expected): ip = '127.0.0.%d' % node - if self.coordinator_counts[ip] != expected: + if self.get_query_count(node) != expected: testcase.fail('Expected %d queries to %s, but got %d. Query counts: %s' % ( expected, ip, self.coordinator_counts[ip], dict(self.coordinator_counts))) @@ -118,7 +122,7 @@ def wait_for_up(cluster, node, wait=True): if host and host.is_up: # BUG: shouldn't have to, but we do if wait: - log.debug("Sleeping 5s until host is up") + log.debug("Sleeping 20s until host is up") time.sleep(20) log.debug("Done waiting for node %s to be up", node) return @@ -132,7 +136,7 @@ def wait_for_down(cluster, node, wait=True): if not host or not host.is_up: # BUG: shouldn't have to, but we do if wait: - log.debug("Sleeping 5s until host is down") + log.debug("Sleeping 10s until host is down") time.sleep(10) log.debug("Done waiting for node %s to be down", node) return From f2113b9a1cf4edb6ced07d5e9bbe429f1511f729 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Fri, 11 Apr 2014 16:40:40 -0500 Subject: [PATCH 31/42] Add note about raising trace timeout to error message --- cassandra/query.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cassandra/query.py b/cassandra/query.py index fc1151f6..ad9248c8 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -427,7 +427,8 @@ class QueryTrace(object): while True: time_spent = time.time() - start if max_wait is not None and time_spent >= max_wait: - raise TraceUnavailable("Trace information was not available within %f seconds" % (max_wait,)) + raise TraceUnavailable( + "Trace information was not available within %f seconds. Consider raising Session.max_trace_wait." % (max_wait,)) log.debug("Attempting to fetch trace info for trace ID: %s", self.trace_id) session_results = self._execute( From eea560e4f6330acd9b5a2e3a4e050402861da923 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Fri, 11 Apr 2014 16:41:08 -0500 Subject: [PATCH 32/42] Preserve all exc info when raising in execute_concurrent() --- cassandra/concurrent.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cassandra/concurrent.py b/cassandra/concurrent.py index 6edd54a1..d036fcc8 100644 --- a/cassandra/concurrent.py +++ b/cassandra/concurrent.py @@ -62,7 +62,12 @@ def execute_concurrent(session, statements_and_parameters, concurrency=100, rais event.wait() if first_error: - raise first_error[0] + exc = first_error[0] + if isinstance(exc, tuple): + (exc_type, value, traceback) = exc + raise exc_type, value, traceback + else: + raise exc else: return results From 9bfd94892fafc9cec89d7625c81614330fa76059 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Fri, 11 Apr 2014 17:05:10 -0500 Subject: [PATCH 33/42] Prettier log message for scheduled retries --- cassandra/pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cassandra/pool.py b/cassandra/pool.py index 2f218af5..df71882b 100644 --- a/cassandra/pool.py +++ b/cassandra/pool.py @@ -228,7 +228,7 @@ class _HostReconnectionHandler(_ReconnectionHandler): if isinstance(exc, AuthenticationFailed): return False else: - log.warn("Error attempting to reconnect to %s, scheduling retry in %f seconds: %s", + log.warn("Error attempting to reconnect to %s, scheduling retry in %s seconds: %s", self.host, next_delay, exc) log.debug("Reconnection error details", exc_info=True) return True From 7186fec6f8a7c7cbf7a6b626d40c1c36f3cb5c1b Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Fri, 11 Apr 2014 17:23:20 -0500 Subject: [PATCH 34/42] Use execute_concurrent_with_args for loadbalancing tests --- tests/integration/long/test_loadbalancingpolicies.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 08b1a155..4545979f 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -27,10 +27,9 @@ class LoadBalancingPolicyTests(unittest.TestCase): def _insert(self, session, keyspace, count=12, consistency_level=ConsistencyLevel.ONE): session.execute('USE %s' % keyspace) - for i in range(count): - ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', - consistency_level=consistency_level) - session.execute(ss) + ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)', + consistency_level=consistency_level) + execute_concurrent_with_args(session, ss, [None] * count) def _query(self, session, keyspace, count=12, consistency_level=ConsistencyLevel.ONE, use_prepared=False): From a975613979b7ad612c91a6d7ec88d211f0820fbc Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Fri, 11 Apr 2014 17:23:52 -0500 Subject: [PATCH 35/42] PEP-8 fixes --- tests/integration/__init__.py | 1 - .../long/test_loadbalancingpolicies.py | 34 +++++++++---------- tests/integration/long/utils.py | 2 +- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 1d67fcf9..239ba2bd 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,4 +1,3 @@ -from ccmlib.common import current_cluster_name import time try: diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 4545979f..11054404 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -2,13 +2,14 @@ import struct from cassandra import ConsistencyLevel, Unavailable from cassandra.cluster import Cluster, NoHostAvailable from cassandra.concurrent import execute_concurrent_with_args -from cassandra.policies import RoundRobinPolicy, DCAwareRoundRobinPolicy, \ - TokenAwarePolicy, WhiteListRoundRobinPolicy +from cassandra.policies import (RoundRobinPolicy, DCAwareRoundRobinPolicy, + TokenAwarePolicy, WhiteListRoundRobinPolicy) from cassandra.query import SimpleStatement from tests.integration import use_multidc, use_singledc -from tests.integration.long.utils import wait_for_up, create_schema, \ - CoordinatorStats, force_stop, wait_for_down, decommission, start, ring, \ - bootstrap, stop, IP_FORMAT +from tests.integration.long.utils import (wait_for_up, create_schema, + CoordinatorStats, force_stop, + wait_for_down, decommission, start, + bootstrap, stop, IP_FORMAT) try: import unittest2 as unittest @@ -48,7 +49,6 @@ class LoadBalancingPolicyTests(unittest.TestCase): routing_key=routing_key) self.coordinator_stats.add_coordinator(session.execute_async(ss)) - def test_roundrobin(self): use_singledc() keyspace = 'test_roundrobin' @@ -90,7 +90,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats.assert_query_count_equals(self, 3, 6) def test_roundrobin_two_dcs(self): - use_multidc([2,2]) + use_multidc([2, 2]) keyspace = 'test_roundrobin_two_dcs' cluster = Cluster( load_balancing_policy=RoundRobinPolicy()) @@ -100,7 +100,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): wait_for_up(cluster, 3, wait=False) wait_for_up(cluster, 4) - create_schema(session, keyspace, replication_strategy=[2,2]) + create_schema(session, keyspace, replication_strategy=[2, 2]) self._insert(session, keyspace) self._query(session, keyspace) @@ -127,7 +127,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats.assert_query_count_equals(self, 5, 3) def test_roundrobin_two_dcs_2(self): - use_multidc([2,2]) + use_multidc([2, 2]) keyspace = 'test_roundrobin_two_dcs_2' cluster = Cluster( load_balancing_policy=RoundRobinPolicy()) @@ -137,7 +137,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): wait_for_up(cluster, 3, wait=False) wait_for_up(cluster, 4) - create_schema(session, keyspace, replication_strategy=[2,2]) + create_schema(session, keyspace, replication_strategy=[2, 2]) self._insert(session, keyspace) self._query(session, keyspace) @@ -164,7 +164,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats.assert_query_count_equals(self, 5, 3) def test_dc_aware_roundrobin_two_dcs(self): - use_multidc([3,2]) + use_multidc([3, 2]) keyspace = 'test_dc_aware_roundrobin_two_dcs' cluster = Cluster( load_balancing_policy=DCAwareRoundRobinPolicy('dc1')) @@ -186,7 +186,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats.assert_query_count_equals(self, 5, 0) def test_dc_aware_roundrobin_two_dcs_2(self): - use_multidc([3,2]) + use_multidc([3, 2]) keyspace = 'test_dc_aware_roundrobin_two_dcs_2' cluster = Cluster( load_balancing_policy=DCAwareRoundRobinPolicy('dc2')) @@ -208,7 +208,7 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats.assert_query_count_equals(self, 5, 6) def test_dc_aware_roundrobin_one_remote_host(self): - use_multidc([2,2]) + use_multidc([2, 2]) keyspace = 'test_dc_aware_roundrobin_one_remote_host' cluster = Cluster( load_balancing_policy=DCAwareRoundRobinPolicy('dc2', used_hosts_per_remote_dc=1)) @@ -250,9 +250,9 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats.assert_query_count_equals(self, 3, 0) self.coordinator_stats.assert_query_count_equals(self, 4, 0) responses = set() - for node in [1,2,5]: + for node in [1, 2, 5]: responses.add(self.coordinator_stats.get_query_count(node)) - self.assertEqual(set([0,0,12]), responses) + self.assertEqual(set([0, 0, 12]), responses) self.coordinator_stats.reset_counts() decommission(5) @@ -264,9 +264,9 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats.assert_query_count_equals(self, 4, 0) self.coordinator_stats.assert_query_count_equals(self, 5, 0) responses = set() - for node in [1,2]: + for node in [1, 2]: responses.add(self.coordinator_stats.get_query_count(node)) - self.assertEqual(set([0,12]), responses) + self.assertEqual(set([0, 12]), responses) self.coordinator_stats.reset_counts() decommission(1) diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index 470b68a0..c765fc96 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -97,7 +97,7 @@ def bootstrap(node, data_center=None, token=None): storage_interface=(IP_FORMAT % node, 7000), jmx_port=str(7000 + 100 * node), remote_debug_port=0, - initial_token=token if token else node*10) + initial_token=token if token else node * 10) get_cluster().add(node_instance, is_seed=False, data_center=data_center) try: From 06949a83dc64681a49851c490b7dc87b3350e930 Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Fri, 11 Apr 2014 17:24:15 -0500 Subject: [PATCH 36/42] Remove unneeded bind() call in example.py --- example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example.py b/example.py index 424613ae..54f8c77f 100755 --- a/example.py +++ b/example.py @@ -56,7 +56,7 @@ def main(): for i in range(10): log.info("inserting row %d" % i) session.execute(query, dict(key="key%d" % i, a='a', b='b')) - session.execute(prepared.bind(("key%d" % i, 'b', 'b'))) + session.execute(prepared, ("key%d" % i, 'b', 'b')) future = session.execute_async("SELECT * FROM mytable") log.info("key\tcol1\tcol2") From 4e9b1ff5bbd635352867d43e69dc213894757db2 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Mon, 14 Apr 2014 20:37:50 -0500 Subject: [PATCH 37/42] Handle Jenkins race conditions --- tests/integration/long/utils.py | 4 ++-- tests/integration/standard/test_metrics.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index c765fc96..d45ab3da 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -122,8 +122,8 @@ def wait_for_up(cluster, node, wait=True): if host and host.is_up: # BUG: shouldn't have to, but we do if wait: - log.debug("Sleeping 20s until host is up") - time.sleep(20) + log.debug("Sleeping 30s until host is up") + time.sleep(30) log.debug("Done waiting for node %s to be up", node) return diff --git a/tests/integration/standard/test_metrics.py b/tests/integration/standard/test_metrics.py index 8f5b3ae7..185c45b1 100644 --- a/tests/integration/standard/test_metrics.py +++ b/tests/integration/standard/test_metrics.py @@ -1,3 +1,5 @@ +import time + try: import unittest2 as unittest except ImportError: @@ -118,6 +120,7 @@ class MetricsTests(unittest.TestCase): # Force kill ccm node get_node(1).stop(wait=True, gently=True) + time.sleep(5) try: # Test write From a82c37f33439e52c0bc288469556337371a398de Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Tue, 15 Apr 2014 15:20:21 -0500 Subject: [PATCH 38/42] Log possible exception for Jenkins --- tests/integration/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 239ba2bd..0b51de26 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,4 +1,5 @@ import time +import traceback try: import unittest2 as unittest @@ -154,6 +155,9 @@ def setup_test_keyspace(): k int PRIMARY KEY, v int )''' session.execute(ddl) + except Exception: + traceback.print_exc() + raise finally: cluster.shutdown() From ef4a4e4a0da3594137cf2af5aa57031ad2b19e72 Mon Sep 17 00:00:00 2001 From: Joaquin Casares Date: Tue, 15 Apr 2014 17:02:05 -0500 Subject: [PATCH 39/42] tearDownClass after loadbalancing tests --- tests/integration/long/test_loadbalancingpolicies.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 11054404..69e22ec5 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -22,7 +22,8 @@ class LoadBalancingPolicyTests(unittest.TestCase): self.coordinator_stats = CoordinatorStats() self.prepared = None - def teardown(self): + @classmethod + def tearDownClass(cls): use_singledc() def _insert(self, session, keyspace, count=12, From 020455c978da9bef02d4cad806157779b838ce1d Mon Sep 17 00:00:00 2001 From: Michael Figuiere Date: Wed, 16 Apr 2014 08:01:03 -0700 Subject: [PATCH 40/42] Add License headers --- benchmarks/base.py | 14 ++++++++++++++ benchmarks/callback_full_pipeline.py | 14 ++++++++++++++ benchmarks/future_batches.py | 14 ++++++++++++++ benchmarks/future_full_pipeline.py | 14 ++++++++++++++ benchmarks/future_full_throttle.py | 14 ++++++++++++++ benchmarks/sync.py | 14 ++++++++++++++ cassandra/__init__.py | 14 ++++++++++++++ cassandra/cluster.py | 14 ++++++++++++++ cassandra/concurrent.py | 14 ++++++++++++++ cassandra/connection.py | 14 ++++++++++++++ cassandra/cqltypes.py | 14 ++++++++++++++ cassandra/decoder.py | 14 ++++++++++++++ cassandra/io/__init__.py | 14 ++++++++++++++ cassandra/io/asyncorereactor.py | 14 ++++++++++++++ cassandra/io/geventreactor.py | 14 ++++++++++++++ cassandra/io/libevreactor.py | 14 ++++++++++++++ cassandra/marshal.py | 14 ++++++++++++++ cassandra/metadata.py | 14 ++++++++++++++ cassandra/metrics.py | 14 ++++++++++++++ cassandra/policies.py | 14 ++++++++++++++ cassandra/pool.py | 14 ++++++++++++++ cassandra/query.py | 14 ++++++++++++++ example.py | 14 ++++++++++++++ ez_setup.py | 14 ++++++++++++++ setup.py | 14 ++++++++++++++ tests/__init__.py | 14 ++++++++++++++ tests/integration/__init__.py | 14 ++++++++++++++ tests/integration/long/__init__.py | 14 ++++++++++++++ tests/integration/long/test_consistency.py | 14 ++++++++++++++ tests/integration/long/test_large_data.py | 14 ++++++++++++++ .../integration/long/test_loadbalancingpolicies.py | 14 ++++++++++++++ tests/integration/long/test_schema.py | 14 ++++++++++++++ tests/integration/long/utils.py | 14 ++++++++++++++ tests/integration/standard/__init__.py | 14 ++++++++++++++ tests/integration/standard/test_cluster.py | 14 ++++++++++++++ tests/integration/standard/test_concurrent.py | 14 ++++++++++++++ tests/integration/standard/test_connection.py | 14 ++++++++++++++ tests/integration/standard/test_factories.py | 14 ++++++++++++++ tests/integration/standard/test_metadata.py | 14 ++++++++++++++ tests/integration/standard/test_metrics.py | 14 ++++++++++++++ .../standard/test_prepared_statements.py | 14 ++++++++++++++ tests/integration/standard/test_query.py | 14 ++++++++++++++ tests/integration/standard/test_types.py | 14 ++++++++++++++ tests/unit/__init__.py | 14 ++++++++++++++ tests/unit/io/__init__.py | 14 ++++++++++++++ tests/unit/io/test_asyncorereactor.py | 14 ++++++++++++++ tests/unit/io/test_libevreactor.py | 14 ++++++++++++++ tests/unit/test_connection.py | 14 ++++++++++++++ tests/unit/test_control_connection.py | 14 ++++++++++++++ tests/unit/test_host_connection_pool.py | 14 ++++++++++++++ tests/unit/test_marshalling.py | 14 ++++++++++++++ tests/unit/test_metadata.py | 14 ++++++++++++++ tests/unit/test_parameter_binding.py | 14 ++++++++++++++ tests/unit/test_policies.py | 14 ++++++++++++++ tests/unit/test_response_future.py | 14 ++++++++++++++ tests/unit/test_types.py | 14 ++++++++++++++ 56 files changed, 784 insertions(+) mode change 100755 => 100644 example.py diff --git a/benchmarks/base.py b/benchmarks/base.py index f00df6b0..d6ed3005 100644 --- a/benchmarks/base.py +++ b/benchmarks/base.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from cProfile import Profile import logging import os.path diff --git a/benchmarks/callback_full_pipeline.py b/benchmarks/callback_full_pipeline.py index 614b1a46..d5e799ea 100644 --- a/benchmarks/callback_full_pipeline.py +++ b/benchmarks/callback_full_pipeline.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from itertools import count diff --git a/benchmarks/future_batches.py b/benchmarks/future_batches.py index 34f9d569..80547964 100644 --- a/benchmarks/future_batches.py +++ b/benchmarks/future_batches.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import Queue diff --git a/benchmarks/future_full_pipeline.py b/benchmarks/future_full_pipeline.py index 843e34a8..9cb98fa0 100644 --- a/benchmarks/future_full_pipeline.py +++ b/benchmarks/future_full_pipeline.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import Queue diff --git a/benchmarks/future_full_throttle.py b/benchmarks/future_full_throttle.py index 684244d2..9c40aead 100644 --- a/benchmarks/future_full_throttle.py +++ b/benchmarks/future_full_throttle.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from base import benchmark, BenchmarkThread diff --git a/benchmarks/sync.py b/benchmarks/sync.py index c6e91ba6..33f2bebb 100644 --- a/benchmarks/sync.py +++ b/benchmarks/sync.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from base import benchmark, BenchmarkThread class Runner(BenchmarkThread): diff --git a/cassandra/__init__.py b/cassandra/__init__.py index abd2c64b..a6cef863 100644 --- a/cassandra/__init__.py +++ b/cassandra/__init__.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging diff --git a/cassandra/cluster.py b/cassandra/cluster.py index 3255f2ce..58c928d9 100644 --- a/cassandra/cluster.py +++ b/cassandra/cluster.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ This module houses the main classes you will interact with, :class:`.Cluster` and :class:`.Session`. diff --git a/cassandra/concurrent.py b/cassandra/concurrent.py index d036fcc8..eb9a526b 100644 --- a/cassandra/concurrent.py +++ b/cassandra/concurrent.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys from itertools import count, cycle diff --git a/cassandra/connection.py b/cassandra/connection.py index 4780f0c9..76675e8b 100644 --- a/cassandra/connection.py +++ b/cassandra/connection.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import errno from functools import wraps, partial import logging diff --git a/cassandra/cqltypes.py b/cassandra/cqltypes.py index 6dbc9beb..0d5c3e6d 100644 --- a/cassandra/cqltypes.py +++ b/cassandra/cqltypes.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Representation of Cassandra data types. These classes should make it simple for the library (and caller software) to deal with Cassandra-style Java class type diff --git a/cassandra/decoder.py b/cassandra/decoder.py index bef543fa..9893c12d 100644 --- a/cassandra/decoder.py +++ b/cassandra/decoder.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from binascii import hexlify import calendar from collections import namedtuple diff --git a/cassandra/io/__init__.py b/cassandra/io/__init__.py index e69de29b..6df2a731 100644 --- a/cassandra/io/__init__.py +++ b/cassandra/io/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/cassandra/io/asyncorereactor.py b/cassandra/io/asyncorereactor.py index c5fcc058..6ed571a7 100644 --- a/cassandra/io/asyncorereactor.py +++ b/cassandra/io/asyncorereactor.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from collections import defaultdict, deque import logging import os diff --git a/cassandra/io/geventreactor.py b/cassandra/io/geventreactor.py index 2d4c75d0..322b9c00 100644 --- a/cassandra/io/geventreactor.py +++ b/cassandra/io/geventreactor.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import gevent from gevent import select, socket from gevent.event import Event diff --git a/cassandra/io/libevreactor.py b/cassandra/io/libevreactor.py index 9826b059..48e87e00 100644 --- a/cassandra/io/libevreactor.py +++ b/cassandra/io/libevreactor.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from collections import defaultdict, deque import logging import os diff --git a/cassandra/marshal.py b/cassandra/marshal.py index b1e17d10..cd3aa177 100644 --- a/cassandra/marshal.py +++ b/cassandra/marshal.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import struct diff --git a/cassandra/metadata.py b/cassandra/metadata.py index 1a0a76bd..147c2e5a 100644 --- a/cassandra/metadata.py +++ b/cassandra/metadata.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from bisect import bisect_right from collections import defaultdict try: diff --git a/cassandra/metrics.py b/cassandra/metrics.py index f5496202..6ed8218b 100644 --- a/cassandra/metrics.py +++ b/cassandra/metrics.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from itertools import chain import logging diff --git a/cassandra/policies.py b/cassandra/policies.py index 1d645781..2d538c2d 100644 --- a/cassandra/policies.py +++ b/cassandra/policies.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from itertools import islice, cycle, groupby, repeat import logging from random import randint diff --git a/cassandra/pool.py b/cassandra/pool.py index df71882b..1e24f3e0 100644 --- a/cassandra/pool.py +++ b/cassandra/pool.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Connection pooling and host management. """ diff --git a/cassandra/query.py b/cassandra/query.py index ad9248c8..0ef2584a 100644 --- a/cassandra/query.py +++ b/cassandra/query.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ This module holds classes for working with prepared statements and specifying consistency levels and retry policies for individual diff --git a/example.py b/example.py old mode 100755 new mode 100644 index 54f8c77f..74637497 --- a/example.py +++ b/example.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/usr/bin/env python import logging diff --git a/ez_setup.py b/ez_setup.py index 25354721..ad6df914 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!python """Bootstrap setuptools installation diff --git a/setup.py b/setup.py index 0c28d3db..f4ecfa5e 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys import ez_setup diff --git a/tests/__init__.py b/tests/__init__.py index d227d78d..97cea29f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging log = logging.getLogger() diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 0b51de26..9475f8ab 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import time import traceback diff --git a/tests/integration/long/__init__.py b/tests/integration/long/__init__.py index e69de29b..6df2a731 100644 --- a/tests/integration/long/__init__.py +++ b/tests/integration/long/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/integration/long/test_consistency.py b/tests/integration/long/test_consistency.py index bf8646a8..b466c5c9 100644 --- a/tests/integration/long/test_consistency.py +++ b/tests/integration/long/test_consistency.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import struct import traceback diff --git a/tests/integration/long/test_large_data.py b/tests/integration/long/test_large_data.py index f50f59aa..184b106d 100644 --- a/tests/integration/long/test_large_data.py +++ b/tests/integration/long/test_large_data.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import Queue from struct import pack import unittest diff --git a/tests/integration/long/test_loadbalancingpolicies.py b/tests/integration/long/test_loadbalancingpolicies.py index 69e22ec5..ca8e8495 100644 --- a/tests/integration/long/test_loadbalancingpolicies.py +++ b/tests/integration/long/test_loadbalancingpolicies.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import struct from cassandra import ConsistencyLevel, Unavailable from cassandra.cluster import Cluster, NoHostAvailable diff --git a/tests/integration/long/test_schema.py b/tests/integration/long/test_schema.py index 4aaf8bff..cc9d3256 100644 --- a/tests/integration/long/test_schema.py +++ b/tests/integration/long/test_schema.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging from cassandra import ConsistencyLevel diff --git a/tests/integration/long/utils.py b/tests/integration/long/utils.py index d45ab3da..eb302ce6 100644 --- a/tests/integration/long/utils.py +++ b/tests/integration/long/utils.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import logging import time diff --git a/tests/integration/standard/__init__.py b/tests/integration/standard/__init__.py index e69de29b..6df2a731 100644 --- a/tests/integration/standard/__init__.py +++ b/tests/integration/standard/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/integration/standard/test_cluster.py b/tests/integration/standard/test_cluster.py index e066eef8..32b5e4f8 100644 --- a/tests/integration/standard/test_cluster.py +++ b/tests/integration/standard/test_cluster.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/integration/standard/test_concurrent.py b/tests/integration/standard/test_concurrent.py index e847cc4e..9fcdfb25 100644 --- a/tests/integration/standard/test_concurrent.py +++ b/tests/integration/standard/test_concurrent.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/integration/standard/test_connection.py b/tests/integration/standard/test_connection.py index 66f4038c..3b585f67 100644 --- a/tests/integration/standard/test_connection.py +++ b/tests/integration/standard/test_connection.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys try: diff --git a/tests/integration/standard/test_factories.py b/tests/integration/standard/test_factories.py index 11fe1659..0f040e4d 100644 --- a/tests/integration/standard/test_factories.py +++ b/tests/integration/standard/test_factories.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/integration/standard/test_metadata.py b/tests/integration/standard/test_metadata.py index fcfa9a37..45e97389 100644 --- a/tests/integration/standard/test_metadata.py +++ b/tests/integration/standard/test_metadata.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/integration/standard/test_metrics.py b/tests/integration/standard/test_metrics.py index 185c45b1..bffb9312 100644 --- a/tests/integration/standard/test_metrics.py +++ b/tests/integration/standard/test_metrics.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import time try: diff --git a/tests/integration/standard/test_prepared_statements.py b/tests/integration/standard/test_prepared_statements.py index acf93ff4..137ad53f 100644 --- a/tests/integration/standard/test_prepared_statements.py +++ b/tests/integration/standard/test_prepared_statements.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/integration/standard/test_query.py b/tests/integration/standard/test_query.py index 36e69f8b..a8d95207 100644 --- a/tests/integration/standard/test_query.py +++ b/tests/integration/standard/test_query.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/integration/standard/test_types.py b/tests/integration/standard/test_types.py index a350c457..ff6fb25c 100644 --- a/tests/integration/standard/test_types.py +++ b/tests/integration/standard/test_types.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index e69de29b..6df2a731 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/unit/io/__init__.py b/tests/unit/io/__init__.py index e69de29b..6df2a731 100644 --- a/tests/unit/io/__init__.py +++ b/tests/unit/io/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/tests/unit/io/test_asyncorereactor.py b/tests/unit/io/test_asyncorereactor.py index 8b5db2e1..6ec297e4 100644 --- a/tests/unit/io/test_asyncorereactor.py +++ b/tests/unit/io/test_asyncorereactor.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/io/test_libevreactor.py b/tests/unit/io/test_libevreactor.py index f7fab9fd..4f54572d 100644 --- a/tests/unit/io/test_libevreactor.py +++ b/tests/unit/io/test_libevreactor.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index 061630ec..12f1fb39 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_control_connection.py b/tests/unit/test_control_connection.py index 157d0dab..2f620ef4 100644 --- a/tests/unit/test_control_connection.py +++ b/tests/unit/test_control_connection.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_host_connection_pool.py b/tests/unit/test_host_connection_pool.py index dd45cd95..48778c39 100644 --- a/tests/unit/test_host_connection_pool.py +++ b/tests/unit/test_host_connection_pool.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_marshalling.py b/tests/unit/test_marshalling.py index 31277e87..c30d1367 100644 --- a/tests/unit/test_marshalling.py +++ b/tests/unit/test_marshalling.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index 5464fb3a..1005e82a 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_parameter_binding.py b/tests/unit/test_parameter_binding.py index 3ed66647..eeeb0331 100644 --- a/tests/unit/test_parameter_binding.py +++ b/tests/unit/test_parameter_binding.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_policies.py b/tests/unit/test_policies.py index 56fd5440..f375e1d5 100644 --- a/tests/unit/test_policies.py +++ b/tests/unit/test_policies.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_response_future.py b/tests/unit/test_response_future.py index ea05a86f..c9a1979f 100644 --- a/tests/unit/test_response_future.py +++ b/tests/unit/test_response_future.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: diff --git a/tests/unit/test_types.py b/tests/unit/test_types.py index e34327a4..b4586a93 100644 --- a/tests/unit/test_types.py +++ b/tests/unit/test_types.py @@ -1,3 +1,17 @@ +# Copyright 2013-2014 DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + try: import unittest2 as unittest except ImportError: From 606bb32069e9752049d5f4f7b6e1a9eac624d51f Mon Sep 17 00:00:00 2001 From: Michael Figuiere Date: Wed, 16 Apr 2014 08:55:27 -0700 Subject: [PATCH 41/42] Fix ez_setup --- ez_setup.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ez_setup.py b/ez_setup.py index ad6df914..25354721 100644 --- a/ez_setup.py +++ b/ez_setup.py @@ -1,17 +1,3 @@ -# Copyright 2013-2014 DataStax, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - #!python """Bootstrap setuptools installation From 2aff6e03dedab5458a96ce7d85d63b91006bca8c Mon Sep 17 00:00:00 2001 From: Tyler Hobbs Date: Wed, 16 Apr 2014 11:28:23 -0500 Subject: [PATCH 42/42] Handle iterators in execute_concurrent --- cassandra/concurrent.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cassandra/concurrent.py b/cassandra/concurrent.py index eb9a526b..88ffd387 100644 --- a/cassandra/concurrent.py +++ b/cassandra/concurrent.py @@ -65,9 +65,16 @@ def execute_concurrent(session, statements_and_parameters, concurrency=100, rais if not statements_and_parameters: return [] + # TODO handle iterators and generators naturally without converting the + # whole thing to a list. This would requires not building a result + # list of Nones up front (we don't know how many results there will be), + # so a dict keyed by index should be used instead. The tricky part is + # knowing when you're the final statement to finish. + statements_and_parameters = list(statements_and_parameters) + event = Event() first_error = [] if raise_on_first_error else None - to_execute = len(statements_and_parameters) # TODO handle iterators/generators + to_execute = len(statements_and_parameters) results = [None] * to_execute num_finished = count(start=1) statements = enumerate(iter(statements_and_parameters))