openstack
/
magnum
Code Issues Proposed changes

Pull metrics by using TLS enabled client 

Magnum introduced TLS support in swarm bay, so swarm monitor needs
to use TLS enabled client to pull data.

In addition, this commit eliminates data pulling from the docker
daemon of individual bay nodes, since they are not secured by TLS.
Instead, we always pull from swarm. Due to that, additional text
parsing is needed, since the output of docker.info from swarm is
not well formed.

Change-Id: I121bfa98622c240d5c777901794a35c0d8990345
Closes-Bug: #1503460
This commit is contained in:
Hongbin Lu 2015-10-07 14:09:40 -04:00
parent 49cf17c12d
commit 30eacb52c9
3 changed files with 77 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
magnum/common/docker_utils.py
View File

@ -73,6 +73,12 @@ def docker_for_container(context, container):
if magnum_utils.is_uuid_like(container):
container = objects.Container.get_by_uuid(context, container)
bay = conductor_utils.retrieve_bay(context, container)
with docker_for_bay(context, bay) as docker:
yield docker
@contextlib.contextmanager
def docker_for_bay(context, bay):
baymodel = conductor_utils.retrieve_baymodel(context, bay)
tcp_url = 'tcp://%s:2376' % bay.api_address

92
magnum/conductor/monitors.py
View File

@ -19,7 +19,7 @@ from oslo_config import cfg
from oslo_log import log
import six
from magnum.conductor.handlers.common import docker_client
from magnum.common import docker_utils
from magnum.i18n import _LW
from magnum import objects
from magnum.objects.fields import BayType as bay_type
@ -39,7 +39,8 @@ CONF.import_opt('default_timeout',
@six.add_metaclass(abc.ABCMeta)
class MonitorBase(object):
def __init__(self, bay):
def __init__(self, context, bay):
self.context = context
self.bay = bay
@abc.abstractproperty
@ -64,8 +65,8 @@ class MonitorBase(object):
class SwarmMonitor(MonitorBase):
def __init__(self, bay):
super(SwarmMonitor, self).__init__(bay)
def __init__(self, context, bay):
super(SwarmMonitor, self).__init__(context, bay)
self.data = {}
self.data['nodes'] = []
self.data['containers'] = []
@ -80,27 +81,23 @@ class SwarmMonitor(MonitorBase):
}
def pull_data(self):
# pull data from each bay node
nodes = []
for node_addr in (self.bay.node_addresses + [self.bay.api_address]):
docker = self._docker_client(node_addr)
node_info = docker.info()
nodes.append(node_info)
self.data['nodes'] = nodes
with docker_utils.docker_for_bay(self.context,
self.bay) as docker:
system_info = docker.info()
self.data['nodes'] = self._parse_node_info(system_info)
# pull data from each container
containers = []
docker = self._docker_swarm_client(self.bay)
for container in docker.containers(all=True):
try:
container = docker.inspect_container(container['Id'])
except Exception as e:
LOG.warn(_LW("Ignore error [%(e)s] when inspecting container "
"%(container_id)s."),
{'e': e, 'container_id': container['Id']},
exc_info=True)
containers.append(container)
self.data['containers'] = containers
# pull data from each container
containers = []
for container in docker.containers(all=True):
try:
container = docker.inspect_container(container['Id'])
except Exception as e:
LOG.warn(_LW("Ignore error [%(e)s] when inspecting "
"container %(container_id)s."),
{'e': e, 'container_id': container['Id']},
exc_info=True)
containers.append(container)
self.data['containers'] = containers
def compute_memory_util(self):
mem_total = 0
@ -115,22 +112,49 @@ class SwarmMonitor(MonitorBase):
else:
return mem_reserved * 100 / mem_total
def _docker_client(self, api_address, port=2375):
tcp_url = 'tcp://%s:%s' % (api_address, port)
return docker_client.DockerHTTPClient(
tcp_url,
CONF.docker.docker_remote_api_version,
CONF.docker.default_timeout
)
def _parse_node_info(self, system_info):
"""Parse system_info to retrieve memory size of each node.
def _docker_swarm_client(self, bay):
return self._docker_client(bay.api_address, port=2376)
:param system_info: The output returned by docker.info(). Example:
{
u'Debug': False,
u'NEventsListener': 0,
u'DriverStatus': [
[u'\x08Strategy', u'spread'],
[u'\x08Filters', u'...'],
[u'\x08Nodes', u'2'],
[u'node1', u'10.0.0.4:2375'],
[u' \u2514 Containers', u'1'],
[u' \u2514 Reserved CPUs', u'0 / 1'],
[u' \u2514 Reserved Memory', u'0 B / 2.052 GiB'],
[u'node2', u'10.0.0.3:2375'],
[u' \u2514 Containers', u'2'],
[u' \u2514 Reserved CPUs', u'0 / 1'],
[u' \u2514 Reserved Memory', u'0 B / 2.052 GiB']
],
u'Containers': 3
}
:return: Memory size of each node. Excample:
[{'MemTotal': 2203318222.848},
{'MemTotal': 2203318222.848}]
"""
nodes = []
for info in system_info['DriverStatus']:
key = info[0]
value = info[1]
if key == u' \u2514 Reserved Memory':
memory = value # Example: '0 B / 2.052 GiB'
memory = memory.split('/')[1].strip() # Example: '2.052 GiB'
memory = memory.split(' ')[0] # Example: '2.052'
memory = float(memory) * 1024 * 1024 * 1024
nodes.append({'MemTotal': memory})
return nodes
def create_monitor(context, bay):
baymodel = objects.BayModel.get_by_uuid(context, bay.baymodel_id)
if baymodel.coe == bay_type.SWARM:
return SwarmMonitor(bay)
return SwarmMonitor(context, bay)
# TODO(hongbin): add support for other bay types
LOG.debug("Cannot create monitor with bay type '%s'" % baymodel.coe)

26
magnum/tests/unit/conductor/test_monitors.py
View File

@ -40,7 +40,7 @@ class MonitorsTestCase(base.TestCase):
bay = utils.get_test_bay(node_addresses=['1.2.3.4'],
api_address='5.6.7.8')
self.bay = objects.Bay(self.context, **bay)
self.monitor = monitors.SwarmMonitor(self.bay)
self.monitor = monitors.SwarmMonitor(self.context, self.bay)
p = mock.patch('magnum.conductor.monitors.SwarmMonitor.metrics_spec',
new_callable=mock.PropertyMock)
self.mock_metrics_spec = p.start()
@ -63,35 +63,35 @@ class MonitorsTestCase(base.TestCase):
monitor = monitors.create_monitor(self.context, self.bay)
self.assertIsNone(monitor)
@mock.patch('magnum.conductor.handlers.common.docker_client.'
'DockerHTTPClient')
def test_swarm_monitor_pull_data_success(self, mock_docker_http_client):
@mock.patch('magnum.common.docker_utils.docker_for_bay')
def test_swarm_monitor_pull_data_success(self, mock_docker_for_bay):
mock_docker = mock.MagicMock()
mock_docker.info.return_value = 'test_node'
mock_docker.info.return_value = {'DriverStatus': [[
u' \u2514 Reserved Memory', u'0 B / 1 GiB']]}
mock_docker.containers.return_value = [mock.MagicMock()]
mock_docker.inspect_container.return_value = 'test_container'
mock_docker_http_client.return_value = mock_docker
mock_docker_for_bay.return_value.__enter__.return_value = mock_docker
self.monitor.pull_data()
self.assertEqual(self.monitor.data['nodes'],
['test_node', 'test_node'])
[{'MemTotal': 1073741824.0}])
self.assertEqual(self.monitor.data['containers'], ['test_container'])
@mock.patch('magnum.conductor.handlers.common.docker_client.'
'DockerHTTPClient')
def test_swarm_monitor_pull_data_raise(self, mock_docker_http_client):
@mock.patch('magnum.common.docker_utils.docker_for_bay')
def test_swarm_monitor_pull_data_raise(self, mock_docker_for_bay):
mock_container = mock.MagicMock()
mock_docker = mock.MagicMock()
mock_docker.info.return_value = 'test_node'
mock_docker.info.return_value = {'DriverStatus': [[
u' \u2514 Reserved Memory', u'0 B / 1 GiB']]}
mock_docker.containers.return_value = [mock_container]
mock_docker.inspect_container.side_effect = Exception("inspect error")
mock_docker_http_client.return_value = mock_docker
mock_docker_for_bay.return_value.__enter__.return_value = mock_docker
self.monitor.pull_data()
self.assertEqual(self.monitor.data['nodes'],
['test_node', 'test_node'])
[{'MemTotal': 1073741824.0}])
self.assertEqual(self.monitor.data['containers'], [mock_container])
def test_swarm_monitor_get_metric_names(self):