openstack
/
magnum
Code Issues Proposed changes

Merge "Add cpu util to K8sMonitor"

This commit is contained in:
Jenkins 2016-04-05 16:44:16 +00:00 committed by Gerrit Code Review
parent e3109fca5b 11307f6ce1
commit 79da93804a
5 changed files with 102 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
magnum/common/exception.py
View File

@ -529,8 +529,8 @@ class MagnumServiceAlreadyExists(Conflict):
message = _("A magnum service with ID %(id)s already exists.")
class UnsupportedK8sMemoryFormat(MagnumException):
message = _("Unsupported memory format for k8s bay.")
class UnsupportedK8sQuantityFormat(MagnumException):
message = _("Unsupported quantity format for k8s bay.")
class FlavorNotFound(ResourceNotFound):

30
magnum/common/utils.py
View File

@ -549,8 +549,18 @@ def raise_exception_invalid_scheme(url):
raise exception.Urllib2InvalidScheme(url=url)
def get_memory_bytes(memory):
"""Kubernetes memory format must be in the format of:
def get_k8s_quantity(quantity):
"""This function is used to get k8s quantity.
It supports to get CPU and Memory quantity:
Kubernetes cpu format must be in the format of:
<signedNumber>'m'
for example:
500m = 0.5 core of cpu
Kubernetes memory format must be in the format of:
<signedNumber><suffix>
signedNumber = digits|digits.digits|digits.|.digits
@ -558,23 +568,29 @@ def get_memory_bytes(memory):
or suffix = E|e<signedNumber>
digits = digit | digit<digits>
digit = 0|1|2|3|4|5|6|7|8|9
:param name: String value of a quantity such as '500m', '1G'
:returns: Quantity number
:raises: exception.UnsupportedK8sQuantityFormat if the quantity string
is a unsupported value
"""
signed_num_regex = r"(^\d+\.\d+)|(^\d+\.)|(\.\d+)|(^\d+)"
matched_signed_number = re.search(signed_num_regex, memory)
matched_signed_number = re.search(signed_num_regex, quantity)
if matched_signed_number is None:
raise exception.UnsupportedK8sMemoryFormat()
raise exception.UnsupportedK8sQuantityFormat()
else:
signed_number = matched_signed_number.group(0)
suffix = memory.replace(signed_number, '', 1)
suffix = quantity.replace(signed_number, '', 1)
if suffix == '':
return float(memory)
return float(quantity)
if re.search(r"^(Ki|Mi|Gi|Ti|Pi|Ei|m|k|M|G|T|P|E|'')$", suffix):
return float(signed_number) * MEMORY_UNITS[suffix]
elif re.search(r"^[E|e][+|-]?(\d+\.\d+$)|(\d+\.$)|(\.\d+$)|(\d+$)",
suffix):
return float(signed_number) * (10 ** float(suffix[1:]))
else:
raise exception.UnsupportedK8sMemoryFormat()
raise exception.UnsupportedK8sQuantityFormat()
def generate_password(length, symbolgroups=None):

61
magnum/conductor/k8s_monitor.py
View File

@ -32,6 +32,10 @@ class K8sMonitor(MonitorBase):
'unit': '%',
'func': 'compute_memory_util',
},
'cpu_util': {
'unit': '%',
'func': 'compute_cpu_util',
},
}
def pull_data(self):
@ -41,22 +45,28 @@ class K8sMonitor(MonitorBase):
pods = k8s_api.list_namespaced_pod('default')
self.data['pods'] = self._parse_pod_info(pods)
def compute_memory_util(self):
mem_total = 0
def _compute_res_util(self, res):
res_total = 0
for node in self.data['nodes']:
mem_total += node['Memory']
mem_reserved = 0
res_total += node[res]
res_reserved = 0
for pod in self.data['pods']:
mem_reserved += pod['Memory']
res_reserved += pod[res]
if mem_total == 0:
if res_total == 0:
return 0
else:
return mem_reserved * 100 / mem_total
return res_reserved * 100 / res_total
def compute_memory_util(self):
return self._compute_res_util('Memory')
def compute_cpu_util(self):
return self._compute_res_util('Cpu')
def _parse_pod_info(self, pods):
"""Parse pods and retrieve memory details about each pod
"""Parse pods and retrieve memory and cpu details about each pod
:param pods: The output of k8s_api.list_namespaced_pods()
For example:
@ -73,7 +83,8 @@ class K8sMonitor(MonitorBase):
'containers': [{
'image': 'nginx',
'resources': {'requests': None,
'limits': "{u'memory': u'1280e3'}"},
'limits': "{u'cpu': u'500m',
u'memory': u'1280e3'}"},
}],
},
'api_version': None,
@ -86,8 +97,8 @@ class K8sMonitor(MonitorBase):
V1PodList object
:return: Memory size of each pod. Example:
[{'Memory': 1280000.0},
{'Memory': 1280000.0}]
[{'Memory': 1280000.0, cpu: 0.5},
{'Memory': 1280000.0, cpu: 0.5}]
"""
pods = pods.items
parsed_containers = []
@ -95,23 +106,27 @@ class K8sMonitor(MonitorBase):
containers = pod.spec.containers
for container in containers:
memory = 0
cpu = 0
resources = container.resources
limits = resources.limits
if limits is not None:
# Output of resources.limits is string
# for example:
# limits = "{'memory': '1000Ki'}"
# limits = "{cpu': '500m': 'memory': '1000Ki'}"
limits = ast.literal_eval(limits)
if limits.get('memory', ''):
memory = utils.get_memory_bytes(limits['memory'])
memory = utils.get_k8s_quantity(limits['memory'])
if limits.get('cpu', ''):
cpu = utils.get_k8s_quantity(limits['cpu'])
container_dict = {
'Memory': memory
'Memory': memory,
'Cpu': cpu,
}
parsed_containers.append(container_dict)
return parsed_containers
def _parse_node_info(self, nodes):
"""Parse nodes to retrieve memory of each node
"""Parse nodes to retrieve memory and cpu of each node
:param nodes: The output of k8s_api.list_namespaced_node()
For example:
@ -119,7 +134,8 @@ class K8sMonitor(MonitorBase):
'items': [{
'status': {
'phase': None,
'capacity': "{u'memory': u'2049852Ki'}",
'capacity': "{u'cpu': u'1',
u'memory': u'2049852Ki'}",
},
},
'api_version': None,
@ -132,9 +148,9 @@ class K8sMonitor(MonitorBase):
magnum.common.pythonk8sclient.swagger_client.models.v1_node_list.
V1NodeList object
:return: Memory size of each node. Excample:
[{'Memory': 1024.0},
{'Memory': 1024.0}]
:return: CPU core number and Memory size of each node. Example:
[{'cpu': 1, 'Memory': 1024.0},
{'cpu': 1, 'Memory': 1024.0}]
"""
nodes = nodes.items
@ -142,9 +158,10 @@ class K8sMonitor(MonitorBase):
for node in nodes:
# Output of node.status.capacity is strong
# for example:
# capacity = "{'memory': '1000Ki'}"
# capacity = "{'cpu': '1', 'memory': '1000Ki'}"
capacity = ast.literal_eval(node.status.capacity)
memory = utils.get_memory_bytes(capacity['memory'])
parsed_nodes.append({'Memory': memory})
memory = utils.get_k8s_quantity(capacity['memory'])
cpu = int(capacity['cpu'])
parsed_nodes.append({'Memory': memory, 'Cpu': cpu})
return parsed_nodes

17
magnum/tests/unit/common/test_utils.py
View File

@ -123,14 +123,15 @@ class UtilsTestCase(base.TestCase):
utils.convert_to_list_dict(['first', 'second'],
'fred'))
def test_get_memory_bytes(self):
self.assertEqual(1024000.0, utils.get_memory_bytes('1000Ki'))
self.assertEqual(0.001, utils.get_memory_bytes('1E-3'))
self.assertEqual(0.5, utils.get_memory_bytes('0.0005k'))
self.assertEqual(1300000.0, utils.get_memory_bytes('1.3E+6'))
self.assertEqual(1300000.0, utils.get_memory_bytes('1.3E6'))
self.assertRaises(exception.UnsupportedK8sMemoryFormat,
utils.get_memory_bytes, '1E1E')
def test_get_k8s_quantity(self):
self.assertEqual(1024000.0, utils.get_k8s_quantity('1000Ki'))
self.assertEqual(0.001, utils.get_k8s_quantity('1E-3'))
self.assertEqual(0.5, utils.get_k8s_quantity('0.0005k'))
self.assertEqual(0.5, utils.get_k8s_quantity('500m'))
self.assertEqual(1300000.0, utils.get_k8s_quantity('1.3E+6'))
self.assertEqual(1300000.0, utils.get_k8s_quantity('1.3E6'))
self.assertRaises(exception.UnsupportedK8sQuantityFormat,
utils.get_k8s_quantity, '1E1E')
class ExecuteTestCase(base.TestCase):

33
magnum/tests/unit/conductor/test_monitors.py
View File

@ -154,7 +154,7 @@ class MonitorsTestCase(base.TestCase):
mock_nodes = mock.MagicMock()
mock_node = mock.MagicMock()
mock_node.status = mock.MagicMock()
mock_node.status.capacity = "{'memory': '2000Ki'}"
mock_node.status.capacity = "{'memory': '2000Ki', 'cpu': '1'}"
mock_nodes.items = [mock_node]
mock_k8s_api.return_value.list_namespaced_node.return_value = (
mock_nodes)
@ -163,16 +163,16 @@ class MonitorsTestCase(base.TestCase):
mock_pod.spec = mock.MagicMock()
mock_container = mock.MagicMock()
mock_container.resources = mock.MagicMock()
mock_container.resources.limits = "{'memory':'100Mi'}"
mock_container.resources.limits = "{'memory': '100Mi', 'cpu': '500m'}"
mock_pod.spec.containers = [mock_container]
mock_pods.items = [mock_pod]
mock_k8s_api.return_value.list_namespaced_pod.return_value = mock_pods
self.k8s_monitor.pull_data()
self.assertEqual(self.k8s_monitor.data['nodes'],
[{'Memory': 2048000.0}])
[{'Memory': 2048000.0, 'Cpu': 1}])
self.assertEqual(self.k8s_monitor.data['pods'],
[{'Memory': 104857600.0}])
[{'Memory': 104857600.0, 'Cpu': 0.5}])
def test_k8s_monitor_get_metric_names(self):
k8s_metric_spec = 'magnum.conductor.k8s_monitor.K8sMonitor.'\
@ -217,6 +217,31 @@ class MonitorsTestCase(base.TestCase):
mem_util = self.k8s_monitor.compute_memory_util()
self.assertEqual(0, mem_util)
def test_k8s_monitor_compute_cpu_util(self):
test_data = {
'nodes': [
{
'Cpu': 1,
},
],
'pods': [
{
'Cpu': 0.5,
},
],
}
self.k8s_monitor.data = test_data
cpu_util = self.k8s_monitor.compute_cpu_util()
self.assertEqual(50, cpu_util)
test_data = {
'nodes': [],
'pods': [],
}
self.k8s_monitor.data = test_data
cpu_util = self.k8s_monitor.compute_cpu_util()
self.assertEqual(0, cpu_util)
def _test_mesos_monitor_pull_data(
self, mock_url_get, state_json, expected_mem_total,
expected_mem_used, expected_cpu_total, expected_cpu_used):