309 lines
13 KiB
Python
309 lines
13 KiB
Python
# Copyright 2013-2015 DataStax, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import struct, time, traceback, sys, logging
|
|
|
|
from cassandra import ConsistencyLevel, OperationTimedOut, ReadTimeout, WriteTimeout, Unavailable
|
|
from cassandra.cluster import Cluster
|
|
from cassandra.policies import TokenAwarePolicy, RoundRobinPolicy, DowngradingConsistencyRetryPolicy
|
|
from cassandra.query import SimpleStatement
|
|
from tests.integration import use_singledc, PROTOCOL_VERSION, execute_until_pass
|
|
|
|
from tests.integration.long.utils import (force_stop, create_schema, wait_for_down, wait_for_up,
|
|
start, CoordinatorStats)
|
|
|
|
try:
|
|
import unittest2 as unittest
|
|
except ImportError:
|
|
import unittest # noqa
|
|
|
|
ALL_CONSISTENCY_LEVELS = set([
|
|
ConsistencyLevel.ANY, ConsistencyLevel.ONE, ConsistencyLevel.TWO,
|
|
ConsistencyLevel.QUORUM, ConsistencyLevel.THREE,
|
|
ConsistencyLevel.ALL, ConsistencyLevel.LOCAL_QUORUM,
|
|
ConsistencyLevel.EACH_QUORUM])
|
|
|
|
MULTI_DC_CONSISTENCY_LEVELS = set([
|
|
ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.EACH_QUORUM])
|
|
|
|
SINGLE_DC_CONSISTENCY_LEVELS = ALL_CONSISTENCY_LEVELS - MULTI_DC_CONSISTENCY_LEVELS
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def setup_module():
|
|
use_singledc()
|
|
|
|
|
|
class ConsistencyTests(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.coordinator_stats = CoordinatorStats()
|
|
|
|
def _cl_failure(self, consistency_level, e):
|
|
self.fail('Instead of success, saw %s for CL.%s:\n\n%s' % (
|
|
e, ConsistencyLevel.value_to_name[consistency_level],
|
|
traceback.format_exc()))
|
|
|
|
def _cl_expected_failure(self, cl):
|
|
self.fail('Test passed at ConsistencyLevel.%s:\n\n%s' % (
|
|
ConsistencyLevel.value_to_name[cl], traceback.format_exc()))
|
|
|
|
def _insert(self, session, keyspace, count, consistency_level=ConsistencyLevel.ONE):
|
|
session.execute('USE %s' % keyspace)
|
|
for i in range(count):
|
|
ss = SimpleStatement('INSERT INTO cf(k, i) VALUES (0, 0)',
|
|
consistency_level=consistency_level)
|
|
execute_until_pass(session, ss)
|
|
|
|
def _query(self, session, keyspace, count, consistency_level=ConsistencyLevel.ONE):
|
|
routing_key = struct.pack('>i', 0)
|
|
for i in range(count):
|
|
ss = SimpleStatement('SELECT * FROM cf WHERE k = 0',
|
|
consistency_level=consistency_level,
|
|
routing_key=routing_key)
|
|
tries = 0
|
|
while True:
|
|
if tries > 100:
|
|
raise RuntimeError("Failed to execute query after 100 attempts: {0}".format(ss))
|
|
try:
|
|
self.coordinator_stats.add_coordinator(session.execute_async(ss))
|
|
break
|
|
except (OperationTimedOut, ReadTimeout):
|
|
ex_type, ex, tb = sys.exc_info()
|
|
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
|
|
del tb
|
|
tries += 1
|
|
time.sleep(1)
|
|
|
|
def _assert_writes_succeed(self, session, keyspace, consistency_levels):
|
|
for cl in consistency_levels:
|
|
self.coordinator_stats.reset_counts()
|
|
try:
|
|
self._insert(session, keyspace, 1, cl)
|
|
except Exception as e:
|
|
self._cl_failure(cl, e)
|
|
|
|
def _assert_reads_succeed(self, session, keyspace, consistency_levels, expected_reader=3):
|
|
for cl in consistency_levels:
|
|
self.coordinator_stats.reset_counts()
|
|
try:
|
|
self._query(session, keyspace, 1, cl)
|
|
for i in range(3):
|
|
if i == expected_reader:
|
|
self.coordinator_stats.assert_query_count_equals(self, i, 1)
|
|
else:
|
|
self.coordinator_stats.assert_query_count_equals(self, i, 0)
|
|
except Exception as e:
|
|
self._cl_failure(cl, e)
|
|
|
|
def _assert_writes_fail(self, session, keyspace, consistency_levels):
|
|
for cl in consistency_levels:
|
|
self.coordinator_stats.reset_counts()
|
|
try:
|
|
self._insert(session, keyspace, 1, cl)
|
|
self._cl_expected_failure(cl)
|
|
except (Unavailable, WriteTimeout):
|
|
pass
|
|
|
|
def _assert_reads_fail(self, session, keyspace, consistency_levels):
|
|
for cl in consistency_levels:
|
|
self.coordinator_stats.reset_counts()
|
|
try:
|
|
self._query(session, keyspace, 1, cl)
|
|
self._cl_expected_failure(cl)
|
|
except (Unavailable, ReadTimeout):
|
|
pass
|
|
|
|
def _test_tokenaware_one_node_down(self, keyspace, rf, accepted):
|
|
cluster = Cluster(
|
|
load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy()),
|
|
protocol_version=PROTOCOL_VERSION)
|
|
session = cluster.connect()
|
|
wait_for_up(cluster, 1, wait=False)
|
|
wait_for_up(cluster, 2)
|
|
|
|
create_schema(session, keyspace, replication_factor=rf)
|
|
self._insert(session, keyspace, count=1)
|
|
self._query(session, keyspace, count=1)
|
|
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
|
|
self.coordinator_stats.assert_query_count_equals(self, 2, 1)
|
|
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
|
|
|
|
try:
|
|
force_stop(2)
|
|
wait_for_down(cluster, 2)
|
|
|
|
self._assert_writes_succeed(session, keyspace, accepted)
|
|
self._assert_reads_succeed(session, keyspace,
|
|
accepted - set([ConsistencyLevel.ANY]))
|
|
self._assert_writes_fail(session, keyspace,
|
|
SINGLE_DC_CONSISTENCY_LEVELS - accepted)
|
|
self._assert_reads_fail(session, keyspace,
|
|
SINGLE_DC_CONSISTENCY_LEVELS - accepted)
|
|
finally:
|
|
start(2)
|
|
wait_for_up(cluster, 2)
|
|
|
|
cluster.shutdown()
|
|
|
|
def test_rfone_tokenaware_one_node_down(self):
|
|
self._test_tokenaware_one_node_down(
|
|
keyspace='test_rfone_tokenaware',
|
|
rf=1,
|
|
accepted=set([ConsistencyLevel.ANY]))
|
|
|
|
def test_rftwo_tokenaware_one_node_down(self):
|
|
self._test_tokenaware_one_node_down(
|
|
keyspace='test_rftwo_tokenaware',
|
|
rf=2,
|
|
accepted=set([ConsistencyLevel.ANY, ConsistencyLevel.ONE]))
|
|
|
|
def test_rfthree_tokenaware_one_node_down(self):
|
|
self._test_tokenaware_one_node_down(
|
|
keyspace='test_rfthree_tokenaware',
|
|
rf=3,
|
|
accepted=set([ConsistencyLevel.ANY, ConsistencyLevel.ONE,
|
|
ConsistencyLevel.TWO, ConsistencyLevel.QUORUM]))
|
|
|
|
def test_rfthree_tokenaware_none_down(self):
|
|
keyspace = 'test_rfthree_tokenaware_none_down'
|
|
cluster = Cluster(
|
|
load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy()),
|
|
protocol_version=PROTOCOL_VERSION)
|
|
session = cluster.connect()
|
|
wait_for_up(cluster, 1, wait=False)
|
|
wait_for_up(cluster, 2)
|
|
|
|
create_schema(session, keyspace, replication_factor=3)
|
|
self._insert(session, keyspace, count=1)
|
|
self._query(session, keyspace, count=1)
|
|
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
|
|
self.coordinator_stats.assert_query_count_equals(self, 2, 1)
|
|
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
|
|
|
|
self.coordinator_stats.reset_counts()
|
|
|
|
self._assert_writes_succeed(session, keyspace, SINGLE_DC_CONSISTENCY_LEVELS)
|
|
self._assert_reads_succeed(session, keyspace,
|
|
SINGLE_DC_CONSISTENCY_LEVELS - set([ConsistencyLevel.ANY]),
|
|
expected_reader=2)
|
|
|
|
cluster.shutdown()
|
|
|
|
def _test_downgrading_cl(self, keyspace, rf, accepted):
|
|
cluster = Cluster(
|
|
load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy()),
|
|
default_retry_policy=DowngradingConsistencyRetryPolicy(),
|
|
protocol_version=PROTOCOL_VERSION)
|
|
session = cluster.connect()
|
|
|
|
create_schema(session, keyspace, replication_factor=rf)
|
|
self._insert(session, keyspace, 1)
|
|
self._query(session, keyspace, 1)
|
|
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
|
|
self.coordinator_stats.assert_query_count_equals(self, 2, 1)
|
|
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
|
|
|
|
try:
|
|
force_stop(2)
|
|
wait_for_down(cluster, 2)
|
|
|
|
self._assert_writes_succeed(session, keyspace, accepted)
|
|
self._assert_reads_succeed(session, keyspace,
|
|
accepted - set([ConsistencyLevel.ANY]))
|
|
self._assert_writes_fail(session, keyspace,
|
|
SINGLE_DC_CONSISTENCY_LEVELS - accepted)
|
|
self._assert_reads_fail(session, keyspace,
|
|
SINGLE_DC_CONSISTENCY_LEVELS - accepted)
|
|
finally:
|
|
start(2)
|
|
wait_for_up(cluster, 2)
|
|
|
|
cluster.shutdown()
|
|
|
|
def test_rfone_downgradingcl(self):
|
|
self._test_downgrading_cl(
|
|
keyspace='test_rfone_downgradingcl',
|
|
rf=1,
|
|
accepted=set([ConsistencyLevel.ANY]))
|
|
|
|
def test_rftwo_downgradingcl(self):
|
|
self._test_downgrading_cl(
|
|
keyspace='test_rftwo_downgradingcl',
|
|
rf=2,
|
|
accepted=SINGLE_DC_CONSISTENCY_LEVELS)
|
|
|
|
def test_rfthree_roundrobin_downgradingcl(self):
|
|
keyspace = 'test_rfthree_roundrobin_downgradingcl'
|
|
cluster = Cluster(
|
|
load_balancing_policy=RoundRobinPolicy(),
|
|
default_retry_policy=DowngradingConsistencyRetryPolicy(),
|
|
protocol_version=PROTOCOL_VERSION)
|
|
self.rfthree_downgradingcl(cluster, keyspace, True)
|
|
|
|
def test_rfthree_tokenaware_downgradingcl(self):
|
|
keyspace = 'test_rfthree_tokenaware_downgradingcl'
|
|
cluster = Cluster(
|
|
load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy()),
|
|
default_retry_policy=DowngradingConsistencyRetryPolicy(),
|
|
protocol_version=PROTOCOL_VERSION)
|
|
self.rfthree_downgradingcl(cluster, keyspace, False)
|
|
|
|
def rfthree_downgradingcl(self, cluster, keyspace, roundrobin):
|
|
session = cluster.connect()
|
|
|
|
create_schema(session, keyspace, replication_factor=2)
|
|
self._insert(session, keyspace, count=12)
|
|
self._query(session, keyspace, count=12)
|
|
|
|
if roundrobin:
|
|
self.coordinator_stats.assert_query_count_equals(self, 1, 4)
|
|
self.coordinator_stats.assert_query_count_equals(self, 2, 4)
|
|
self.coordinator_stats.assert_query_count_equals(self, 3, 4)
|
|
else:
|
|
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
|
|
self.coordinator_stats.assert_query_count_equals(self, 2, 12)
|
|
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
|
|
|
|
try:
|
|
self.coordinator_stats.reset_counts()
|
|
force_stop(2)
|
|
wait_for_down(cluster, 2)
|
|
|
|
self._assert_writes_succeed(session, keyspace, SINGLE_DC_CONSISTENCY_LEVELS)
|
|
|
|
# Test reads that expected to complete successfully
|
|
for cl in SINGLE_DC_CONSISTENCY_LEVELS - set([ConsistencyLevel.ANY]):
|
|
self.coordinator_stats.reset_counts()
|
|
self._query(session, keyspace, 12, consistency_level=cl)
|
|
if roundrobin:
|
|
self.coordinator_stats.assert_query_count_equals(self, 1, 6)
|
|
self.coordinator_stats.assert_query_count_equals(self, 2, 0)
|
|
self.coordinator_stats.assert_query_count_equals(self, 3, 6)
|
|
else:
|
|
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
|
|
self.coordinator_stats.assert_query_count_equals(self, 2, 0)
|
|
self.coordinator_stats.assert_query_count_equals(self, 3, 12)
|
|
finally:
|
|
start(2)
|
|
wait_for_up(cluster, 2)
|
|
|
|
session.cluster.shutdown()
|
|
|
|
# TODO: can't be done in this class since we reuse the ccm cluster
|
|
# instead we should create these elsewhere
|
|
# def test_rfthree_downgradingcl_twodcs(self):
|
|
# def test_rfthree_downgradingcl_twodcs_dcaware(self):
|