diff --git a/quark/agent/agent.py b/quark/agent/agent.py index 8d991ee..d287bda 100644 --- a/quark/agent/agent.py +++ b/quark/agent/agent.py @@ -86,10 +86,9 @@ def partition_vifs(xapi_client, interfaces, security_group_states): return added, updated, removed -def ack_groups(groups): +def ack_groups(client, groups): if len(groups) > 0: - write_groups_client = sg_cli.SecurityGroupsClient(use_master=True) - write_groups_client.update_group_states_for_vifs(groups, True) + client.update_group_states_for_vifs(groups, True) def run(): @@ -120,7 +119,7 @@ def run(): sg_states) xapi_client.update_interfaces(new_sg, updated_sg, removed_sg) groups_to_ack = [v for v in new_sg + updated_sg if v.success] - ack_groups(groups_to_ack) + ack_groups(groups_client, groups_to_ack) except Exception: LOG.exception("Unable to get security groups from registry and " diff --git a/quark/cache/redis_base.py b/quark/cache/redis_base.py index 7f5e222..7d24e01 100644 --- a/quark/cache/redis_base.py +++ b/quark/cache/redis_base.py @@ -15,17 +15,16 @@ import functools import json -from random import shuffle import string import netaddr from oslo_config import cfg from oslo_log import log as logging -import redis -import redis.sentinel from quark import exceptions as q_exc +from twiceredis import TwiceRedis + CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -33,18 +32,6 @@ MAC_TRANS_TABLE = string.maketrans(string.ascii_uppercase, string.ascii_lowercase) quark_opts = [ - cfg.StrOpt('redis_host', - default='127.0.0.1', - help=_("The server to write redis data to or" - " retrieve sentinel information from, as appropriate")), - cfg.IntOpt('redis_port', - default=6379, - help=_("The port for the redis server to write redis data to or" - " retrieve sentinel information from, as appropriate")), - cfg.BoolOpt("redis_use_sentinels", - default=False, - help=_("Tell the redis client to use sentinels rather than a " - "direct connection")), cfg.ListOpt("redis_sentinel_hosts", default=["localhost:26397"], help=_("Comma-separated list of host:port pairs for Redis " @@ -78,81 +65,26 @@ def handle_connection_error(fn): def wrapped(*args, **kwargs): try: return fn(*args, **kwargs) - except redis.ConnectionError as e: + except TwiceRedis.generic_error as e: LOG.exception(e) raise q_exc.RedisConnectionFailure() return wrapped class ClientBase(object): - read_connection_pool = None - write_connection_pool = None + def __init__(self): + self._client = self.get_redis_client() - def __init__(self, use_master=False): - self._use_master = use_master - try: - if CONF.QUARK.redis_use_sentinels: - self._compile_sentinel_list() - self._ensure_connection_pools_exist() - self._client = self._client() - except redis.ConnectionError as e: - LOG.exception(e) - raise q_exc.RedisConnectionFailure() + def get_redis_client(self): + sentinels = [tuple(str.split(host_pair, ':')) + for host_pair in CONF.QUARK.redis_sentinel_hosts] - def _prepare_pool_connection(self, master): - LOG.info("Creating redis connection pool for the first time...") - host = CONF.QUARK.redis_host - port = CONF.QUARK.redis_port - - connect_args = [] - connect_kw = {} - if CONF.QUARK.redis_password: - connect_kw["password"] = CONF.QUARK.redis_password - - if CONF.QUARK.redis_use_sentinels: - LOG.info("Using redis sentinel connections %s" % - self._sentinel_list) - klass = redis.sentinel.SentinelConnectionPool - connect_args.append(CONF.QUARK.redis_sentinel_master) - connect_args.append(redis.sentinel.Sentinel(self._sentinel_list)) - connect_kw["check_connection"] = True - connect_kw["is_master"] = master - else: - LOG.info("Using redis host %s:%s" % (host, port)) - klass = redis.ConnectionPool - connect_kw["host"] = host - connect_kw["port"] = port - - return klass, connect_args, connect_kw - - def _ensure_connection_pools_exist(self): - if not (ClientBase.write_connection_pool or - ClientBase.read_connection_pool): - klass, args, kwargs = self._prepare_pool_connection(master=False) - ClientBase.read_connection_pool = klass(*args, **kwargs) - - klass, args, kwargs = self._prepare_pool_connection(master=True) - ClientBase.write_connection_pool = klass(*args, **kwargs) - - def _compile_sentinel_list(self): - self._sentinel_list = [tuple(host.split(':')) - for host in CONF.QUARK.redis_sentinel_hosts] - # NOTE(asadoughi): RM12113: shuffle list of sentinels to distribute - # connection load - shuffle(self._sentinel_list) - if not self._sentinel_list: - raise TypeError("sentinel_list is not a properly formatted" - "list of 'host:port' pairs") - - def _client(self): - pool = ClientBase.read_connection_pool - if self._use_master: - pool = ClientBase.write_connection_pool - - kwargs = {"connection_pool": pool, - "db": CONF.QUARK.redis_db, - "socket_timeout": CONF.QUARK.redis_socket_timeout} - return redis.StrictRedis(**kwargs) + return TwiceRedis(master_name=CONF.QUARK.redis_sentinel_master, + sentinels=sentinels, + password=CONF.QUARK.redis_password, + check_connection=True, + socket_timeout=CONF.QUARK.redis_socket_timeout, + min_other_sentinels=2) def vif_key(self, device_id, mac_address): mac = str(netaddr.EUI(mac_address)) @@ -162,55 +94,66 @@ class ClientBase(object): return "{0}.{1}".format(device_id, mac) @handle_connection_error - def echo(self, echo_str): - return self._client.echo(echo_str) + def ping(self): + # NOTE(tr3buchet): if this gets used by anything other than the + # redis_sg_tool, self._client.disconnect() + # needs to be called before returning + return self._client.master.ping() and self._client.slave.ping() @handle_connection_error def vif_keys(self, field=None): - keys = self._client.keys("*.????????????") + keys = self._client.slave.keys("*.????????????") filtered = [] if isinstance(keys, str): keys = [keys] - for key in keys: - value = None - if field: - value = self._client.hget(key, field) - else: - value = self._client.hgetall(key) + with self._client.slave.pipeline() as pipe: + for key in keys: + value = None + if field: + value = pipe.hget(key, field) + else: + value = pipe.hgetall(key) + values = pipe.execute() + for value in values: if value: filtered.append(key) return filtered @handle_connection_error def set_field(self, key, field, data): - return self.set_field_raw(key, field, json.dumps(data)) + self.set_field_raw(key, field, json.dumps(data)) @handle_connection_error def set_field_raw(self, key, field, data): - return self._client.hset(key, field, data) + self._client.master.hset(key, field, data) + self._client.master.disconnect() @handle_connection_error def get_field(self, key, field): - return self._client.hget(key, field) + return self._client.slave.hget(key, field) @handle_connection_error def delete_field(self, key, field): - return self._client.hdel(key, field) + self._client.master.hdel(key, field) + self._client.master.disconnect() @handle_connection_error def delete_key(self, key): - return self._client.delete(key) + self._client.master.delete(key) + self._client.master.disconnect() @handle_connection_error def get_fields(self, keys, field): - p = self._client.pipeline() - for key in keys: - p.hget(key, field) - return p.execute() + with self._client.slave.pipeline() as pipe: + for key in keys: + pipe.hget(key, field) + values = pipe.execute() + return values @handle_connection_error def set_fields(self, keys, field, value): - p = self._client.pipeline() - for key in keys: - p.hset(key, field, value) - return p.execute() + with self._client.master.pipeline() as pipe: + for key in keys: + pipe.hset(key, field, value) + pipe.execute() + self._client.master.disconnect() diff --git a/quark/cache/security_groups_client.py b/quark/cache/security_groups_client.py index f85a486..60871e9 100644 --- a/quark/cache/security_groups_client.py +++ b/quark/cache/security_groups_client.py @@ -19,7 +19,6 @@ import netaddr from oslo_log import log as logging from quark.cache import redis_base -from quark import exceptions as q_exc from quark import protocols from quark import utils @@ -144,8 +143,6 @@ class SecurityGroupsClient(redis_base.ClientBase): """Writes a series of security group rules to a redis server.""" LOG.info("Applying security group rules for device %s with MAC %s" % (device_id, mac_address)) - if not self._use_master: - raise q_exc.RedisSlaveWritesForbidden() rule_dict = {SECURITY_GROUP_RULE_KEY: rules} redis_key = self.vif_key(device_id, mac_address) @@ -200,9 +197,6 @@ class SecurityGroupsClient(redis_base.ClientBase): @utils.retry_loop(3) def update_group_states_for_vifs(self, vifs, ack): """Updates security groups by setting the ack field""" - if not self._use_master: - raise q_exc.RedisSlaveWritesForbidden() - vif_keys = [self.vif_key(vif.device_id, vif.mac_address) for vif in vifs] self.set_fields(vif_keys, SECURITY_GROUP_ACK, ack) diff --git a/quark/drivers/security_groups.py b/quark/drivers/security_groups.py index 60e0c81..068d759 100644 --- a/quark/drivers/security_groups.py +++ b/quark/drivers/security_groups.py @@ -24,7 +24,7 @@ LOG = logging.getLogger(__name__) class SecurityGroupDriver(object): @env.has_capability(env.Capabilities.SECURITY_GROUPS) def update_port(self, **kwargs): - client = sg_client.SecurityGroupsClient(use_master=True) + client = sg_client.SecurityGroupsClient() if "security_groups" in kwargs: if kwargs["security_groups"]: payload = client.serialize_groups( @@ -38,7 +38,7 @@ class SecurityGroupDriver(object): @env.has_capability(env.Capabilities.SECURITY_GROUPS) def delete_port(self, **kwargs): - client = sg_client.SecurityGroupsClient(use_master=True) + client = sg_client.SecurityGroupsClient() try: client.delete_vif(kwargs["device_id"], kwargs["mac_address"]) diff --git a/quark/exceptions.py b/quark/exceptions.py index 12d05a1..a997199 100644 --- a/quark/exceptions.py +++ b/quark/exceptions.py @@ -130,10 +130,6 @@ class RedisConnectionFailure(exceptions.NeutronException): message = _("No connection to Redis could be made.") -class RedisSlaveWritesForbidden(exceptions.NeutronException): - message = _("No write actions can be applied to Slave redis nodes.") - - class NoBackendConnectionsDefined(exceptions.NeutronException): message = _("This driver cannot be used without a backend connection " "definition. %(msg)") diff --git a/quark/tests/cache/test_redis_base.py b/quark/tests/cache/test_redis_base.py deleted file mode 100644 index 022c118..0000000 --- a/quark/tests/cache/test_redis_base.py +++ /dev/null @@ -1,265 +0,0 @@ -# Copyright 2014 Openstack Foundation -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# - -import contextlib -import json -import uuid - -import mock -import netaddr -from oslo_config import cfg -import redis - -from quark.cache import redis_base -from quark import exceptions as q_exc -from quark.tests import test_base - -CONF = cfg.CONF - - -class TestClientBase(test_base.TestBase): - - def setUp(self): - super(TestClientBase, self).setUp() - # Forces the connection pool to be recreated on every test - redis_base.ClientBase.read_connection_pool = None - redis_base.ClientBase.write_connection_pool = None - - @mock.patch("quark.cache.redis_base.redis") - def test_vif_key(self, *args, **kwargs): - client = redis_base.ClientBase() - device_id = str(uuid.uuid4()) - mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF") - - redis_key = client.vif_key(device_id, mac_address.value) - expected = "%s.%s" % (device_id, "aabbccddeeff") - self.assertEqual(expected, redis_key) - - @mock.patch("redis.ConnectionPool") - @mock.patch("quark.cache.redis_base.redis.StrictRedis") - def test_init(self, strict_redis, conn_pool): - host = "127.0.0.1" - port = 6379 - redis_base.ClientBase() - conn_pool.assert_called_with(host=host, port=port) - self.assertIsNotNone(redis_base.ClientBase.read_connection_pool) - self.assertIsNotNone(redis_base.ClientBase.write_connection_pool) - - @mock.patch("redis.ConnectionPool") - def test_client_connection_fails_gracefully(self, conn_pool): - conn_err = redis.ConnectionError - with mock.patch("redis.StrictRedis") as redis_mock: - redis_mock.side_effect = conn_err - with self.assertRaises(q_exc.RedisConnectionFailure): - redis_base.ClientBase(use_master=True) - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_get_field(self, strict_redis): - rc = redis_base.ClientBase() - mock_client = rc._client = mock.MagicMock() - mock_client.hget.return_value = "returned hash field" - - r = rc.get_field("1.000000000002", "test_field_name") - - mock_client.hget.assert_called_once_with("1.000000000002", - "test_field_name") - - self.assertEqual(r, "returned hash field") - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_vif_keys_hget(self, strict_redis): - rc = redis_base.ClientBase() - keys = ['1.000000000002', '2.000000000003'] - mock_client = rc._client = mock.MagicMock() - mock_client.hget.return_value = "returned hash field" - mock_client.keys.return_value = keys - - r = rc.vif_keys(field="test_field_name") - - mock_client.hget.assert_has_calls( - [mock.call("1.000000000002", "test_field_name"), - mock.call("2.000000000003", "test_field_name")]) - self.assertFalse(mock_client.hgetall.called) - self.assertEqual(r, keys) - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_vif_keys_hget_string_key_returned(self, strict_redis): - rc = redis_base.ClientBase() - keys = '1.000000000002' - mock_client = rc._client = mock.MagicMock() - mock_client.hget.return_value = "returned hash field" - mock_client.keys.return_value = keys - - r = rc.vif_keys(field="test_field_name") - - mock_client.hget.assert_called_once_with("1.000000000002", - "test_field_name") - self.assertEqual(r, [keys]) - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_vif_keys_hget_nil_returned(self, strict_redis): - rc = redis_base.ClientBase() - keys = ['1.000000000002', '2.000000000003'] - mock_client = rc._client = mock.MagicMock() - mock_client.hget.side_effect = ["returned hash field", None] - mock_client.keys.return_value = keys - - r = rc.vif_keys(field="test_field_name") - - mock_client.hget.assert_has_calls( - [mock.call("1.000000000002", "test_field_name"), - mock.call("2.000000000003", "test_field_name")]) - self.assertFalse(mock_client.hgetall.called) - self.assertEqual(r, keys[:1]) - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_vif_keys_hgetall(self, strict_redis): - rc = redis_base.ClientBase() - keys = ['1.000000000002', '2.000000000003'] - mock_client = rc._client = mock.MagicMock() - mock_client.hgetall.return_value = { - "returned hash field1": "returned hash value1", - "returned hash field2": "returned hash value2" - } - mock_client.keys.return_value = keys - - r = rc.vif_keys() - - mock_client.hgetall.assert_has_calls([mock.call("1.000000000002"), - mock.call("2.000000000003")]) - self.assertFalse(mock_client.hget.called) - self.assertEqual(r, keys) - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_vif_keys_hgetall_nil_returned(self, strict_redis): - rc = redis_base.ClientBase() - keys = ['1.000000000002', '2.000000000003'] - mock_client = rc._client = mock.MagicMock() - mock_client.hgetall.side_effect = [ - { - "returned hash field1": "returned hash value1", - "returned hash field2": "returned hash value2" - }, - None - ] - mock_client.keys.return_value = keys - - r = rc.vif_keys() - - mock_client.hgetall.assert_has_calls([mock.call("1.000000000002"), - mock.call("2.000000000003")]) - self.assertFalse(mock_client.hget.called) - self.assertEqual(r, keys[:1]) - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_set_field(self, strict_redis): - rc = redis_base.ClientBase(use_master=True) - mock_client = rc._client = mock.MagicMock() - dummy_data = {"dummy_data": "foo"} - - rc.set_field("1.000000000002", "test_field_name", dummy_data) - - mock_client.hset.assert_called_once_with("1.000000000002", - "test_field_name", - json.dumps(dummy_data)) - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_delete_field(self, strict_redis): - rc = redis_base.ClientBase(use_master=True) - mock_client = rc._client = mock.MagicMock() - rc.delete_field("1.000000000002", "test_field_name") - - mock_client.hdel.assert_called_once_with("1.000000000002", - "test_field_name") - - @mock.patch( - "quark.cache.redis_base.redis.StrictRedis") - def test_get_fields(self, strict_redis): - mock_redis = mock.MagicMock() - mock_pipeline = mock.MagicMock() - strict_redis.return_value = mock_redis - mock_redis.pipeline.return_value = mock_pipeline - mock_pipeline.execute.return_value = "returned executed" - rc = redis_base.ClientBase() - - r = rc.get_fields(["1.000000000002", "1.000000000002", - "5.000000000006", "7.000000000008"], - "test_field_name") - - mock_pipeline.hget.assert_has_calls( - [mock.call("1.000000000002", "test_field_name"), - mock.call("1.000000000002", "test_field_name"), - mock.call("5.000000000006", "test_field_name"), - mock.call("7.000000000008", "test_field_name")], - any_order=True) - - mock_pipeline.execute.assert_called_once_with() - self.assertEqual(r, "returned executed") - - -class TestRedisSentinelConnection(test_base.TestBase): - def setUp(self): - super(TestRedisSentinelConnection, self).setUp() - # Forces the connection pool to be recreated on every test - redis_base.ClientBase.read_connection_pool = None - redis_base.ClientBase.write_connection_pool = None - - @contextlib.contextmanager - def _stubs(self, use_sentinels, sentinels, master_label): - CONF.set_override("redis_use_sentinels", True, "QUARK") - CONF.set_override("redis_sentinel_hosts", sentinels, "QUARK") - CONF.set_override("redis_sentinel_master", master_label, "QUARK") - yield - CONF.set_override("redis_use_sentinels", False, "QUARK") - CONF.set_override("redis_sentinel_hosts", '', "QUARK") - CONF.set_override("redis_sentinel_master", '', "QUARK") - - @mock.patch("redis.sentinel.Sentinel") - @mock.patch("redis.sentinel.SentinelConnectionPool") - @mock.patch("redis.sentinel.Sentinel.master_for") - @mock.patch("quark.cache.redis_base.redis.StrictRedis") - def test_sentinel_connection(self, strict_redis, master_for, - sentinel_pool, sentinel_mock): - host = "127.0.0.1" - port = 6379 - sentinels = ["%s:%s" % (host, port)] - master_label = "master" - sentinel_mock.return_value = sentinels - - with self._stubs(True, sentinels, master_label): - redis_base.ClientBase(use_master=True) - sentinel_pool.assert_called_with(master_label, sentinels, - check_connection=True, - is_master=True) - - @mock.patch("redis.sentinel.SentinelConnectionPool") - @mock.patch("redis.sentinel.Sentinel.master_for") - @mock.patch("quark.cache.redis_base.redis.StrictRedis") - def test_sentinel_connection_bad_format_raises(self, strict_redis, - master_for, sentinel_pool): - sentinels = "" - master_label = "master" - - with self._stubs(True, sentinels, master_label): - with self.assertRaises(TypeError): - redis_base.ClientBase(is_master=True) diff --git a/quark/tests/cache/test_security_groups_client.py b/quark/tests/cache/test_security_groups_client.py index 0f8830c..c26ba2d 100644 --- a/quark/tests/cache/test_security_groups_client.py +++ b/quark/tests/cache/test_security_groups_client.py @@ -33,85 +33,66 @@ class TestRedisSecurityGroupsClient(test_base.TestBase): def setUp(self): super(TestRedisSecurityGroupsClient, self).setUp() - # Forces the connection pool to be recreated on every test - sg_client.SecurityGroupsClient.connection_pool = None @mock.patch("uuid.uuid4") - @mock.patch("redis.ConnectionPool") - @mock.patch("quark.cache.redis_base.redis.StrictRedis") - def test_apply_rules(self, strict_redis, conn_pool, uuid4): - client = sg_client.SecurityGroupsClient(use_master=True) + @mock.patch("quark.cache.redis_base.TwiceRedis") + def test_apply_rules(self, strict_redis, uuid4): + client = sg_client.SecurityGroupsClient() device_id = "device" uuid4.return_value = "uuid" mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF") client.apply_rules(device_id, mac_address.value, []) - self.assertTrue(client._client.hset.called) + self.assertTrue(client._client.master.hset.called) redis_key = client.vif_key(device_id, mac_address.value) rule_dict = {"rules": []} - client._client.hset.assert_any_call( + client._client.master.hset.assert_any_call( redis_key, sg_client.SECURITY_GROUP_HASH_ATTR, json.dumps(rule_dict)) - client._client.hset.assert_any_call( + client._client.master.hset.assert_any_call( redis_key, sg_client.SECURITY_GROUP_ACK, False) @mock.patch("uuid.uuid4") - @mock.patch("redis.ConnectionPool") - @mock.patch("quark.cache.redis_base.redis.StrictRedis") - def test_delete_vif(self, strict_redis, conn_pool, uuid4): - client = sg_client.SecurityGroupsClient(use_master=True) + @mock.patch("quark.cache.redis_base.TwiceRedis") + def test_delete_vif(self, strict_redis, uuid4): + client = sg_client.SecurityGroupsClient() device_id = "device" uuid4.return_value = "uuid" mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF") redis_key = client.vif_key(device_id, mac_address.value) client.delete_vif(device_id, mac_address) - client._client.delete.assert_called_with(redis_key) + client._client.master.delete.assert_called_with(redis_key) - @mock.patch("redis.ConnectionPool") - @mock.patch( - "quark.cache.security_groups_client.SecurityGroupsClient.vif_key") - @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_apply_rules_with_slave_fails(self, strict_redis, vif_key, - conn_pool): - client = sg_client.SecurityGroupsClient() - port_id = 1 - mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF") - with self.assertRaises(q_exc.RedisSlaveWritesForbidden): - client.apply_rules(port_id, mac_address.value, []) - - @mock.patch("redis.ConnectionPool") - def test_apply_rules_set_fails_gracefully(self, conn_pool): + def test_apply_rules_set_fails_gracefully(self): port_id = 1 mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF") conn_err = redis.ConnectionError - with mock.patch("redis.StrictRedis") as redis_mock: + with mock.patch("quark.cache.security_groups_client." + "redis_base.ClientBase") as redis_mock: mocked_redis_cli = mock.MagicMock() redis_mock.return_value = mocked_redis_cli - client = sg_client.SecurityGroupsClient(use_master=True) - mocked_redis_cli.hset.side_effect = conn_err + client = sg_client.SecurityGroupsClient() + mocked_redis_cli.master.hset.side_effect = conn_err with self.assertRaises(q_exc.RedisConnectionFailure): client.apply_rules(port_id, mac_address.value, []) - @mock.patch("redis.ConnectionPool") @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_serialize_group_no_rules(self, strict_redis, conn_pool): + "quark.cache.security_groups_client.redis_base.TwiceRedis") + def test_serialize_group_no_rules(self, strict_redis): client = sg_client.SecurityGroupsClient() group = models.SecurityGroup() payload = client.serialize_groups([group]) self.assertEqual([], payload) - @mock.patch("redis.ConnectionPool") @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_serialize_group_with_rules(self, strict_redis, conn_pool): + "quark.cache.security_groups_client.redis_base.TwiceRedis") + def test_serialize_group_with_rules(self, strict_redis): rule_dict = {"ethertype": 0x800, "protocol": 6, "port_range_min": 80, "port_range_max": 443, "direction": "ingress"} client = sg_client.SecurityGroupsClient() @@ -131,11 +112,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase): self.assertEqual("", rule["source network"]) self.assertEqual("", rule["destination network"]) - @mock.patch("redis.ConnectionPool") @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_serialize_group_with_rules_and_remote_network(self, strict_redis, - conn_pool): + "quark.cache.security_groups_client.redis_base.TwiceRedis") + def test_serialize_group_with_rules_and_remote_network(self, strict_redis): rule_dict = {"ethertype": 0x800, "protocol": 1, "direction": "ingress", "remote_ip_prefix": "192.168.0.0/24"} client = sg_client.SecurityGroupsClient() @@ -155,10 +134,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase): self.assertEqual("::ffff:192.168.0.0/120", rule["source network"]) self.assertEqual("", rule["destination network"]) - @mock.patch("redis.ConnectionPool") @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_serialize_group_egress_rules(self, strict_redis, conn_pool): + "quark.cache.security_groups_client.redis_base.TwiceRedis") + def test_serialize_group_egress_rules(self, strict_redis): rule_dict = {"ethertype": 0x800, "protocol": 1, "direction": "egress", "remote_ip_prefix": "192.168.0.0/24"} @@ -179,10 +157,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase): self.assertEqual("::ffff:192.168.0.0/120", rule["destination network"]) self.assertEqual("", rule["source network"]) - @mock.patch("redis.ConnectionPool") @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_serialize_filters_source_v4_net(self, strict_redis, conn_pool): + "quark.cache.security_groups_client.redis_base.TwiceRedis") + def test_serialize_filters_source_v4_net(self, strict_redis): rule_dict = {"ethertype": 0x800, "protocol": 1, "direction": "ingress", "remote_ip_prefix": "192.168.0.0/0"} client = sg_client.SecurityGroupsClient() @@ -202,10 +179,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase): self.assertEqual("", rule["source network"]) self.assertEqual("", rule["destination network"]) - @mock.patch("redis.ConnectionPool") @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_serialize_filters_source_v6_net(self, strict_redis, conn_pool): + "quark.cache.security_groups_client.redis_base.TwiceRedis") + def test_serialize_filters_source_v6_net(self, strict_redis): rule_dict = {"ethertype": 0x86DD, "protocol": 58, "direction": "ingress", "remote_ip_prefix": "feed::/0"} @@ -226,10 +202,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase): self.assertEqual("", rule["source network"]) self.assertEqual("", rule["destination network"]) - @mock.patch("redis.ConnectionPool") @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_serialize_filters_dest_v4_net(self, strict_redis, conn_pool): + "quark.cache.security_groups_client.redis_base.TwiceRedis") + def test_serialize_filters_dest_v4_net(self, strict_redis): rule_dict = {"ethertype": 0x800, "protocol": 1, "direction": "egress", "remote_ip_prefix": "192.168.0.0/0"} client = sg_client.SecurityGroupsClient() @@ -249,10 +224,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase): self.assertEqual("", rule["source network"]) self.assertEqual("", rule["destination network"]) - @mock.patch("redis.ConnectionPool") @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_serialize_filters_dest_v6_net_(self, strict_redis, conn_pool): + "quark.cache.security_groups_client.redis_base.TwiceRedis") + def test_serialize_filters_dest_v6_net_(self, strict_redis): rule_dict = {"ethertype": 0x86DD, "protocol": 58, "direction": "egress", "remote_ip_prefix": "feed::/0"} @@ -279,24 +253,19 @@ class TestRedisForAgent(test_base.TestBase): super(TestRedisForAgent, self).setUp() patch = mock.patch("quark.cache.security_groups_client.redis_base." - "redis.StrictRedis") + "TwiceRedis") self.MockSentinel = patch.start() self.addCleanup(patch.stop) @mock.patch( - "quark.cache.security_groups_client.redis_base.redis.StrictRedis") - def test_get_security_group_states_empty(self, strict_redis): - mock_redis = mock.MagicMock() - mock_pipeline = mock.MagicMock() - strict_redis.return_value = mock_redis - mock_redis.pipeline.return_value = mock_pipeline - + "quark.cache.security_groups_client.SecurityGroupsClient.get_fields") + def test_get_security_group_states_empty(self, mock_get_fields): rc = sg_client.SecurityGroupsClient() - group_uuids = rc.get_security_group_states(set()) - mock_redis.pipeline.assert_called_once_with() - self.assertEqual(mock_pipeline.get.call_count, 0) - mock_pipeline.execute.assert_called_once_with() - self.assertEqual(group_uuids, {}) + mock_get_fields.return_value = [] + group_states = rc.get_security_group_states([]) + mock_get_fields.assert_called_once_with([], + sg_client.SECURITY_GROUP_ACK) + self.assertEqual(group_states, {}) @mock.patch( "quark.cache.security_groups_client.SecurityGroupsClient.get_fields") diff --git a/quark/tools/redis_sg_tool.py b/quark/tools/redis_sg_tool.py index 828bf1f..41550eb 100755 --- a/quark/tools/redis_sg_tool.py +++ b/quark/tools/redis_sg_tool.py @@ -100,12 +100,10 @@ class QuarkRedisTool(object): print("Redis security groups tool. Re-run with -h/--help for " "options") - def _get_connection(self, use_master=False, giveup=True): - client = sg_client.SecurityGroupsClient(use_master=use_master) + def _get_connection(self, giveup=True): + client = sg_client.SecurityGroupsClient() try: - # You have to use the connection determine it's functional - result = client.echo("connected") - if result == "connected": + if client.ping(): return client except Exception as e: print(e) diff --git a/requirements.txt b/requirements.txt index 9191a62..492df56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ Routes==2.1 aiclib==0.84 gunicorn==19.3.0 pymysql==0.6.6 -redis==2.10.3 +twiceredis>=1.0.6 docopt==0.6.2 # agent deps