x/quark
Code Issues Proposed changes

twiceredis optimization

Browse Source
This commit is contained in:
Trey Morris
2015-07-27 17:22:01 -05:00
parent 8dc27e7495
commit c80afc298d
9 changed files with 95 additions and 461 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
quark/agent/agent.py
View File

@@ -86,10 +86,9 @@ def partition_vifs(xapi_client, interfaces, security_group_states):
return added, updated, removed
def ack_groups(groups):
def ack_groups(client, groups):
if len(groups) > 0:
write_groups_client = sg_cli.SecurityGroupsClient(use_master=True)
write_groups_client.update_group_states_for_vifs(groups, True)
client.update_group_states_for_vifs(groups, True)
def run():
@@ -120,7 +119,7 @@ def run():
sg_states)
xapi_client.update_interfaces(new_sg, updated_sg, removed_sg)
groups_to_ack = [v for v in new_sg + updated_sg if v.success]
ack_groups(groups_to_ack)
ack_groups(groups_client, groups_to_ack)
except Exception:
LOG.exception("Unable to get security groups from registry and "

151
quark/cache/redis_base.py vendored
View File

@@ -15,17 +15,16 @@
import functools
import json
from random import shuffle
import string
import netaddr
from oslo_config import cfg
from oslo_log import log as logging
import redis
import redis.sentinel
from quark import exceptions as q_exc
from twiceredis import TwiceRedis
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@@ -33,18 +32,6 @@ MAC_TRANS_TABLE = string.maketrans(string.ascii_uppercase,
string.ascii_lowercase)
quark_opts = [
cfg.StrOpt('redis_host',
default='127.0.0.1',
help=_("The server to write redis data to or"
" retrieve sentinel information from, as appropriate")),
cfg.IntOpt('redis_port',
default=6379,
help=_("The port for the redis server to write redis data to or"
" retrieve sentinel information from, as appropriate")),
cfg.BoolOpt("redis_use_sentinels",
default=False,
help=_("Tell the redis client to use sentinels rather than a "
"direct connection")),
cfg.ListOpt("redis_sentinel_hosts",
default=["localhost:26397"],
help=_("Comma-separated list of host:port pairs for Redis "
@@ -78,81 +65,26 @@ def handle_connection_error(fn):
def wrapped(*args, **kwargs):
try:
return fn(*args, **kwargs)
except redis.ConnectionError as e:
except TwiceRedis.generic_error as e:
LOG.exception(e)
raise q_exc.RedisConnectionFailure()
return wrapped
class ClientBase(object):
read_connection_pool = None
write_connection_pool = None
def __init__(self):
self._client = self.get_redis_client()
def __init__(self, use_master=False):
self._use_master = use_master
try:
if CONF.QUARK.redis_use_sentinels:
self._compile_sentinel_list()
self._ensure_connection_pools_exist()
self._client = self._client()
except redis.ConnectionError as e:
LOG.exception(e)
raise q_exc.RedisConnectionFailure()
def get_redis_client(self):
sentinels = [tuple(str.split(host_pair, ':'))
for host_pair in CONF.QUARK.redis_sentinel_hosts]
def _prepare_pool_connection(self, master):
LOG.info("Creating redis connection pool for the first time...")
host = CONF.QUARK.redis_host
port = CONF.QUARK.redis_port
connect_args = []
connect_kw = {}
if CONF.QUARK.redis_password:
connect_kw["password"] = CONF.QUARK.redis_password
if CONF.QUARK.redis_use_sentinels:
LOG.info("Using redis sentinel connections %s" %
self._sentinel_list)
klass = redis.sentinel.SentinelConnectionPool
connect_args.append(CONF.QUARK.redis_sentinel_master)
connect_args.append(redis.sentinel.Sentinel(self._sentinel_list))
connect_kw["check_connection"] = True
connect_kw["is_master"] = master
else:
LOG.info("Using redis host %s:%s" % (host, port))
klass = redis.ConnectionPool
connect_kw["host"] = host
connect_kw["port"] = port
return klass, connect_args, connect_kw
def _ensure_connection_pools_exist(self):
if not (ClientBase.write_connection_pool or
ClientBase.read_connection_pool):
klass, args, kwargs = self._prepare_pool_connection(master=False)
ClientBase.read_connection_pool = klass(*args, **kwargs)
klass, args, kwargs = self._prepare_pool_connection(master=True)
ClientBase.write_connection_pool = klass(*args, **kwargs)
def _compile_sentinel_list(self):
self._sentinel_list = [tuple(host.split(':'))
for host in CONF.QUARK.redis_sentinel_hosts]
# NOTE(asadoughi): RM12113: shuffle list of sentinels to distribute
# connection load
shuffle(self._sentinel_list)
if not self._sentinel_list:
raise TypeError("sentinel_list is not a properly formatted"
"list of 'host:port' pairs")
def _client(self):
pool = ClientBase.read_connection_pool
if self._use_master:
pool = ClientBase.write_connection_pool
kwargs = {"connection_pool": pool,
"db": CONF.QUARK.redis_db,
"socket_timeout": CONF.QUARK.redis_socket_timeout}
return redis.StrictRedis(**kwargs)
return TwiceRedis(master_name=CONF.QUARK.redis_sentinel_master,
sentinels=sentinels,
password=CONF.QUARK.redis_password,
check_connection=True,
socket_timeout=CONF.QUARK.redis_socket_timeout,
min_other_sentinels=2)
def vif_key(self, device_id, mac_address):
mac = str(netaddr.EUI(mac_address))
@@ -162,55 +94,66 @@ class ClientBase(object):
return "{0}.{1}".format(device_id, mac)
@handle_connection_error
def echo(self, echo_str):
return self._client.echo(echo_str)
def ping(self):
# NOTE(tr3buchet): if this gets used by anything other than the
# redis_sg_tool, self._client.disconnect()
# needs to be called before returning
return self._client.master.ping() and self._client.slave.ping()
@handle_connection_error
def vif_keys(self, field=None):
keys = self._client.keys("*.????????????")
keys = self._client.slave.keys("*.????????????")
filtered = []
if isinstance(keys, str):
keys = [keys]
for key in keys:
value = None
if field:
value = self._client.hget(key, field)
else:
value = self._client.hgetall(key)
with self._client.slave.pipeline() as pipe:
for key in keys:
value = None
if field:
value = pipe.hget(key, field)
else:
value = pipe.hgetall(key)
values = pipe.execute()
for value in values:
if value:
filtered.append(key)
return filtered
@handle_connection_error
def set_field(self, key, field, data):
return self.set_field_raw(key, field, json.dumps(data))
self.set_field_raw(key, field, json.dumps(data))
@handle_connection_error
def set_field_raw(self, key, field, data):
return self._client.hset(key, field, data)
self._client.master.hset(key, field, data)
self._client.master.disconnect()
@handle_connection_error
def get_field(self, key, field):
return self._client.hget(key, field)
return self._client.slave.hget(key, field)
@handle_connection_error
def delete_field(self, key, field):
return self._client.hdel(key, field)
self._client.master.hdel(key, field)
self._client.master.disconnect()
@handle_connection_error
def delete_key(self, key):
return self._client.delete(key)
self._client.master.delete(key)
self._client.master.disconnect()
@handle_connection_error
def get_fields(self, keys, field):
p = self._client.pipeline()
for key in keys:
p.hget(key, field)
return p.execute()
with self._client.slave.pipeline() as pipe:
for key in keys:
pipe.hget(key, field)
values = pipe.execute()
return values
@handle_connection_error
def set_fields(self, keys, field, value):
p = self._client.pipeline()
for key in keys:
p.hset(key, field, value)
return p.execute()
with self._client.master.pipeline() as pipe:
for key in keys:
pipe.hset(key, field, value)
pipe.execute()
self._client.master.disconnect()

6
quark/cache/security_groups_client.py vendored
View File

@@ -19,7 +19,6 @@ import netaddr
from oslo_log import log as logging
from quark.cache import redis_base
from quark import exceptions as q_exc
from quark import protocols
from quark import utils
@@ -144,8 +143,6 @@ class SecurityGroupsClient(redis_base.ClientBase):
"""Writes a series of security group rules to a redis server."""
LOG.info("Applying security group rules for device %s with MAC %s" %
(device_id, mac_address))
if not self._use_master:
raise q_exc.RedisSlaveWritesForbidden()
rule_dict = {SECURITY_GROUP_RULE_KEY: rules}
redis_key = self.vif_key(device_id, mac_address)
@@ -200,9 +197,6 @@ class SecurityGroupsClient(redis_base.ClientBase):
@utils.retry_loop(3)
def update_group_states_for_vifs(self, vifs, ack):
"""Updates security groups by setting the ack field"""
if not self._use_master:
raise q_exc.RedisSlaveWritesForbidden()
vif_keys = [self.vif_key(vif.device_id, vif.mac_address)
for vif in vifs]
self.set_fields(vif_keys, SECURITY_GROUP_ACK, ack)

4
quark/drivers/security_groups.py
View File

@@ -24,7 +24,7 @@ LOG = logging.getLogger(__name__)
class SecurityGroupDriver(object):
@env.has_capability(env.Capabilities.SECURITY_GROUPS)
def update_port(self, **kwargs):
client = sg_client.SecurityGroupsClient(use_master=True)
client = sg_client.SecurityGroupsClient()
if "security_groups" in kwargs:
if kwargs["security_groups"]:
payload = client.serialize_groups(
@@ -38,7 +38,7 @@ class SecurityGroupDriver(object):
@env.has_capability(env.Capabilities.SECURITY_GROUPS)
def delete_port(self, **kwargs):
client = sg_client.SecurityGroupsClient(use_master=True)
client = sg_client.SecurityGroupsClient()
try:
client.delete_vif(kwargs["device_id"],
kwargs["mac_address"])

4
quark/exceptions.py
View File

@@ -130,10 +130,6 @@ class RedisConnectionFailure(exceptions.NeutronException):
message = _("No connection to Redis could be made.")
class RedisSlaveWritesForbidden(exceptions.NeutronException):
message = _("No write actions can be applied to Slave redis nodes.")
class NoBackendConnectionsDefined(exceptions.NeutronException):
message = _("This driver cannot be used without a backend connection "
"definition. %(msg)")

265
quark/tests/cache/test_redis_base.py vendored
View File

@@ -1,265 +0,0 @@
# Copyright 2014 Openstack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
#
import contextlib
import json
import uuid
import mock
import netaddr
from oslo_config import cfg
import redis
from quark.cache import redis_base
from quark import exceptions as q_exc
from quark.tests import test_base
CONF = cfg.CONF
class TestClientBase(test_base.TestBase):
def setUp(self):
super(TestClientBase, self).setUp()
# Forces the connection pool to be recreated on every test
redis_base.ClientBase.read_connection_pool = None
redis_base.ClientBase.write_connection_pool = None
@mock.patch("quark.cache.redis_base.redis")
def test_vif_key(self, *args, **kwargs):
client = redis_base.ClientBase()
device_id = str(uuid.uuid4())
mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF")
redis_key = client.vif_key(device_id, mac_address.value)
expected = "%s.%s" % (device_id, "aabbccddeeff")
self.assertEqual(expected, redis_key)
@mock.patch("redis.ConnectionPool")
@mock.patch("quark.cache.redis_base.redis.StrictRedis")
def test_init(self, strict_redis, conn_pool):
host = "127.0.0.1"
port = 6379
redis_base.ClientBase()
conn_pool.assert_called_with(host=host, port=port)
self.assertIsNotNone(redis_base.ClientBase.read_connection_pool)
self.assertIsNotNone(redis_base.ClientBase.write_connection_pool)
@mock.patch("redis.ConnectionPool")
def test_client_connection_fails_gracefully(self, conn_pool):
conn_err = redis.ConnectionError
with mock.patch("redis.StrictRedis") as redis_mock:
redis_mock.side_effect = conn_err
with self.assertRaises(q_exc.RedisConnectionFailure):
redis_base.ClientBase(use_master=True)
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_get_field(self, strict_redis):
rc = redis_base.ClientBase()
mock_client = rc._client = mock.MagicMock()
mock_client.hget.return_value = "returned hash field"
r = rc.get_field("1.000000000002", "test_field_name")
mock_client.hget.assert_called_once_with("1.000000000002",
"test_field_name")
self.assertEqual(r, "returned hash field")
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_vif_keys_hget(self, strict_redis):
rc = redis_base.ClientBase()
keys = ['1.000000000002', '2.000000000003']
mock_client = rc._client = mock.MagicMock()
mock_client.hget.return_value = "returned hash field"
mock_client.keys.return_value = keys
r = rc.vif_keys(field="test_field_name")
mock_client.hget.assert_has_calls(
[mock.call("1.000000000002", "test_field_name"),
mock.call("2.000000000003", "test_field_name")])
self.assertFalse(mock_client.hgetall.called)
self.assertEqual(r, keys)
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_vif_keys_hget_string_key_returned(self, strict_redis):
rc = redis_base.ClientBase()
keys = '1.000000000002'
mock_client = rc._client = mock.MagicMock()
mock_client.hget.return_value = "returned hash field"
mock_client.keys.return_value = keys
r = rc.vif_keys(field="test_field_name")
mock_client.hget.assert_called_once_with("1.000000000002",
"test_field_name")
self.assertEqual(r, [keys])
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_vif_keys_hget_nil_returned(self, strict_redis):
rc = redis_base.ClientBase()
keys = ['1.000000000002', '2.000000000003']
mock_client = rc._client = mock.MagicMock()
mock_client.hget.side_effect = ["returned hash field", None]
mock_client.keys.return_value = keys
r = rc.vif_keys(field="test_field_name")
mock_client.hget.assert_has_calls(
[mock.call("1.000000000002", "test_field_name"),
mock.call("2.000000000003", "test_field_name")])
self.assertFalse(mock_client.hgetall.called)
self.assertEqual(r, keys[:1])
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_vif_keys_hgetall(self, strict_redis):
rc = redis_base.ClientBase()
keys = ['1.000000000002', '2.000000000003']
mock_client = rc._client = mock.MagicMock()
mock_client.hgetall.return_value = {
"returned hash field1": "returned hash value1",
"returned hash field2": "returned hash value2"
}
mock_client.keys.return_value = keys
r = rc.vif_keys()
mock_client.hgetall.assert_has_calls([mock.call("1.000000000002"),
mock.call("2.000000000003")])
self.assertFalse(mock_client.hget.called)
self.assertEqual(r, keys)
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_vif_keys_hgetall_nil_returned(self, strict_redis):
rc = redis_base.ClientBase()
keys = ['1.000000000002', '2.000000000003']
mock_client = rc._client = mock.MagicMock()
mock_client.hgetall.side_effect = [
{
"returned hash field1": "returned hash value1",
"returned hash field2": "returned hash value2"
},
None
]
mock_client.keys.return_value = keys
r = rc.vif_keys()
mock_client.hgetall.assert_has_calls([mock.call("1.000000000002"),
mock.call("2.000000000003")])
self.assertFalse(mock_client.hget.called)
self.assertEqual(r, keys[:1])
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_set_field(self, strict_redis):
rc = redis_base.ClientBase(use_master=True)
mock_client = rc._client = mock.MagicMock()
dummy_data = {"dummy_data": "foo"}
rc.set_field("1.000000000002", "test_field_name", dummy_data)
mock_client.hset.assert_called_once_with("1.000000000002",
"test_field_name",
json.dumps(dummy_data))
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_delete_field(self, strict_redis):
rc = redis_base.ClientBase(use_master=True)
mock_client = rc._client = mock.MagicMock()
rc.delete_field("1.000000000002", "test_field_name")
mock_client.hdel.assert_called_once_with("1.000000000002",
"test_field_name")
@mock.patch(
"quark.cache.redis_base.redis.StrictRedis")
def test_get_fields(self, strict_redis):
mock_redis = mock.MagicMock()
mock_pipeline = mock.MagicMock()
strict_redis.return_value = mock_redis
mock_redis.pipeline.return_value = mock_pipeline
mock_pipeline.execute.return_value = "returned executed"
rc = redis_base.ClientBase()
r = rc.get_fields(["1.000000000002", "1.000000000002",
"5.000000000006", "7.000000000008"],
"test_field_name")
mock_pipeline.hget.assert_has_calls(
[mock.call("1.000000000002", "test_field_name"),
mock.call("1.000000000002", "test_field_name"),
mock.call("5.000000000006", "test_field_name"),
mock.call("7.000000000008", "test_field_name")],
any_order=True)
mock_pipeline.execute.assert_called_once_with()
self.assertEqual(r, "returned executed")
class TestRedisSentinelConnection(test_base.TestBase):
def setUp(self):
super(TestRedisSentinelConnection, self).setUp()
# Forces the connection pool to be recreated on every test
redis_base.ClientBase.read_connection_pool = None
redis_base.ClientBase.write_connection_pool = None
@contextlib.contextmanager
def _stubs(self, use_sentinels, sentinels, master_label):
CONF.set_override("redis_use_sentinels", True, "QUARK")
CONF.set_override("redis_sentinel_hosts", sentinels, "QUARK")
CONF.set_override("redis_sentinel_master", master_label, "QUARK")
yield
CONF.set_override("redis_use_sentinels", False, "QUARK")
CONF.set_override("redis_sentinel_hosts", '', "QUARK")
CONF.set_override("redis_sentinel_master", '', "QUARK")
@mock.patch("redis.sentinel.Sentinel")
@mock.patch("redis.sentinel.SentinelConnectionPool")
@mock.patch("redis.sentinel.Sentinel.master_for")
@mock.patch("quark.cache.redis_base.redis.StrictRedis")
def test_sentinel_connection(self, strict_redis, master_for,
sentinel_pool, sentinel_mock):
host = "127.0.0.1"
port = 6379
sentinels = ["%s:%s" % (host, port)]
master_label = "master"
sentinel_mock.return_value = sentinels
with self._stubs(True, sentinels, master_label):
redis_base.ClientBase(use_master=True)
sentinel_pool.assert_called_with(master_label, sentinels,
check_connection=True,
is_master=True)
@mock.patch("redis.sentinel.SentinelConnectionPool")
@mock.patch("redis.sentinel.Sentinel.master_for")
@mock.patch("quark.cache.redis_base.redis.StrictRedis")
def test_sentinel_connection_bad_format_raises(self, strict_redis,
master_for, sentinel_pool):
sentinels = ""
master_label = "master"
with self._stubs(True, sentinels, master_label):
with self.assertRaises(TypeError):
redis_base.ClientBase(is_master=True)

109
quark/tests/cache/test_security_groups_client.py vendored
View File

@@ -33,85 +33,66 @@ class TestRedisSecurityGroupsClient(test_base.TestBase):
def setUp(self):
super(TestRedisSecurityGroupsClient, self).setUp()
# Forces the connection pool to be recreated on every test
sg_client.SecurityGroupsClient.connection_pool = None
@mock.patch("uuid.uuid4")
@mock.patch("redis.ConnectionPool")
@mock.patch("quark.cache.redis_base.redis.StrictRedis")
def test_apply_rules(self, strict_redis, conn_pool, uuid4):
client = sg_client.SecurityGroupsClient(use_master=True)
@mock.patch("quark.cache.redis_base.TwiceRedis")
def test_apply_rules(self, strict_redis, uuid4):
client = sg_client.SecurityGroupsClient()
device_id = "device"
uuid4.return_value = "uuid"
mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF")
client.apply_rules(device_id, mac_address.value, [])
self.assertTrue(client._client.hset.called)
self.assertTrue(client._client.master.hset.called)
redis_key = client.vif_key(device_id, mac_address.value)
rule_dict = {"rules": []}
client._client.hset.assert_any_call(
client._client.master.hset.assert_any_call(
redis_key, sg_client.SECURITY_GROUP_HASH_ATTR,
json.dumps(rule_dict))
client._client.hset.assert_any_call(
client._client.master.hset.assert_any_call(
redis_key, sg_client.SECURITY_GROUP_ACK, False)
@mock.patch("uuid.uuid4")
@mock.patch("redis.ConnectionPool")
@mock.patch("quark.cache.redis_base.redis.StrictRedis")
def test_delete_vif(self, strict_redis, conn_pool, uuid4):
client = sg_client.SecurityGroupsClient(use_master=True)
@mock.patch("quark.cache.redis_base.TwiceRedis")
def test_delete_vif(self, strict_redis, uuid4):
client = sg_client.SecurityGroupsClient()
device_id = "device"
uuid4.return_value = "uuid"
mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF")
redis_key = client.vif_key(device_id, mac_address.value)
client.delete_vif(device_id, mac_address)
client._client.delete.assert_called_with(redis_key)
client._client.master.delete.assert_called_with(redis_key)
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.SecurityGroupsClient.vif_key")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_apply_rules_with_slave_fails(self, strict_redis, vif_key,
conn_pool):
client = sg_client.SecurityGroupsClient()
port_id = 1
mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF")
with self.assertRaises(q_exc.RedisSlaveWritesForbidden):
client.apply_rules(port_id, mac_address.value, [])
@mock.patch("redis.ConnectionPool")
def test_apply_rules_set_fails_gracefully(self, conn_pool):
def test_apply_rules_set_fails_gracefully(self):
port_id = 1
mac_address = netaddr.EUI("AA:BB:CC:DD:EE:FF")
conn_err = redis.ConnectionError
with mock.patch("redis.StrictRedis") as redis_mock:
with mock.patch("quark.cache.security_groups_client."
"redis_base.ClientBase") as redis_mock:
mocked_redis_cli = mock.MagicMock()
redis_mock.return_value = mocked_redis_cli
client = sg_client.SecurityGroupsClient(use_master=True)
mocked_redis_cli.hset.side_effect = conn_err
client = sg_client.SecurityGroupsClient()
mocked_redis_cli.master.hset.side_effect = conn_err
with self.assertRaises(q_exc.RedisConnectionFailure):
client.apply_rules(port_id, mac_address.value, [])
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_serialize_group_no_rules(self, strict_redis, conn_pool):
"quark.cache.security_groups_client.redis_base.TwiceRedis")
def test_serialize_group_no_rules(self, strict_redis):
client = sg_client.SecurityGroupsClient()
group = models.SecurityGroup()
payload = client.serialize_groups([group])
self.assertEqual([], payload)
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_serialize_group_with_rules(self, strict_redis, conn_pool):
"quark.cache.security_groups_client.redis_base.TwiceRedis")
def test_serialize_group_with_rules(self, strict_redis):
rule_dict = {"ethertype": 0x800, "protocol": 6, "port_range_min": 80,
"port_range_max": 443, "direction": "ingress"}
client = sg_client.SecurityGroupsClient()
@@ -131,11 +112,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase):
self.assertEqual("", rule["source network"])
self.assertEqual("", rule["destination network"])
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_serialize_group_with_rules_and_remote_network(self, strict_redis,
conn_pool):
"quark.cache.security_groups_client.redis_base.TwiceRedis")
def test_serialize_group_with_rules_and_remote_network(self, strict_redis):
rule_dict = {"ethertype": 0x800, "protocol": 1, "direction": "ingress",
"remote_ip_prefix": "192.168.0.0/24"}
client = sg_client.SecurityGroupsClient()
@@ -155,10 +134,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase):
self.assertEqual("::ffff:192.168.0.0/120", rule["source network"])
self.assertEqual("", rule["destination network"])
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_serialize_group_egress_rules(self, strict_redis, conn_pool):
"quark.cache.security_groups_client.redis_base.TwiceRedis")
def test_serialize_group_egress_rules(self, strict_redis):
rule_dict = {"ethertype": 0x800, "protocol": 1,
"direction": "egress",
"remote_ip_prefix": "192.168.0.0/24"}
@@ -179,10 +157,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase):
self.assertEqual("::ffff:192.168.0.0/120", rule["destination network"])
self.assertEqual("", rule["source network"])
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_serialize_filters_source_v4_net(self, strict_redis, conn_pool):
"quark.cache.security_groups_client.redis_base.TwiceRedis")
def test_serialize_filters_source_v4_net(self, strict_redis):
rule_dict = {"ethertype": 0x800, "protocol": 1, "direction": "ingress",
"remote_ip_prefix": "192.168.0.0/0"}
client = sg_client.SecurityGroupsClient()
@@ -202,10 +179,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase):
self.assertEqual("", rule["source network"])
self.assertEqual("", rule["destination network"])
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_serialize_filters_source_v6_net(self, strict_redis, conn_pool):
"quark.cache.security_groups_client.redis_base.TwiceRedis")
def test_serialize_filters_source_v6_net(self, strict_redis):
rule_dict = {"ethertype": 0x86DD, "protocol": 58,
"direction": "ingress",
"remote_ip_prefix": "feed::/0"}
@@ -226,10 +202,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase):
self.assertEqual("", rule["source network"])
self.assertEqual("", rule["destination network"])
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_serialize_filters_dest_v4_net(self, strict_redis, conn_pool):
"quark.cache.security_groups_client.redis_base.TwiceRedis")
def test_serialize_filters_dest_v4_net(self, strict_redis):
rule_dict = {"ethertype": 0x800, "protocol": 1, "direction": "egress",
"remote_ip_prefix": "192.168.0.0/0"}
client = sg_client.SecurityGroupsClient()
@@ -249,10 +224,9 @@ class TestRedisSecurityGroupsClient(test_base.TestBase):
self.assertEqual("", rule["source network"])
self.assertEqual("", rule["destination network"])
@mock.patch("redis.ConnectionPool")
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_serialize_filters_dest_v6_net_(self, strict_redis, conn_pool):
"quark.cache.security_groups_client.redis_base.TwiceRedis")
def test_serialize_filters_dest_v6_net_(self, strict_redis):
rule_dict = {"ethertype": 0x86DD, "protocol": 58,
"direction": "egress",
"remote_ip_prefix": "feed::/0"}
@@ -279,24 +253,19 @@ class TestRedisForAgent(test_base.TestBase):
super(TestRedisForAgent, self).setUp()
patch = mock.patch("quark.cache.security_groups_client.redis_base."
"redis.StrictRedis")
"TwiceRedis")
self.MockSentinel = patch.start()
self.addCleanup(patch.stop)
@mock.patch(
"quark.cache.security_groups_client.redis_base.redis.StrictRedis")
def test_get_security_group_states_empty(self, strict_redis):
mock_redis = mock.MagicMock()
mock_pipeline = mock.MagicMock()
strict_redis.return_value = mock_redis
mock_redis.pipeline.return_value = mock_pipeline
"quark.cache.security_groups_client.SecurityGroupsClient.get_fields")
def test_get_security_group_states_empty(self, mock_get_fields):
rc = sg_client.SecurityGroupsClient()
group_uuids = rc.get_security_group_states(set())
mock_redis.pipeline.assert_called_once_with()
self.assertEqual(mock_pipeline.get.call_count, 0)
mock_pipeline.execute.assert_called_once_with()
self.assertEqual(group_uuids, {})
mock_get_fields.return_value = []
group_states = rc.get_security_group_states([])
mock_get_fields.assert_called_once_with([],
sg_client.SECURITY_GROUP_ACK)
self.assertEqual(group_states, {})
@mock.patch(
"quark.cache.security_groups_client.SecurityGroupsClient.get_fields")

8
quark/tools/redis_sg_tool.py
View File

@@ -100,12 +100,10 @@ class QuarkRedisTool(object):
print("Redis security groups tool. Re-run with -h/--help for "
"options")
def _get_connection(self, use_master=False, giveup=True):
client = sg_client.SecurityGroupsClient(use_master=use_master)
def _get_connection(self, giveup=True):
client = sg_client.SecurityGroupsClient()
try:
# You have to use the connection determine it's functional
result = client.echo("connected")
if result == "connected":
if client.ping():
return client
except Exception as e:
print(e)

2
requirements.txt
View File

@@ -9,7 +9,7 @@ Routes==2.1
aiclib==0.84
gunicorn==19.3.0
pymysql==0.6.6
redis==2.10.3
twiceredis>=1.0.6
docopt==0.6.2
# agent deps