diff --git a/magnum/common/exception.py b/magnum/common/exception.py index c807eea3c6..d8273c783b 100644 --- a/magnum/common/exception.py +++ b/magnum/common/exception.py @@ -529,8 +529,8 @@ class MagnumServiceAlreadyExists(Conflict): message = _("A magnum service with ID %(id)s already exists.") -class UnsupportedK8sMemoryFormat(MagnumException): - message = _("Unsupported memory format for k8s bay.") +class UnsupportedK8sQuantityFormat(MagnumException): + message = _("Unsupported quantity format for k8s bay.") class FlavorNotFound(ResourceNotFound): diff --git a/magnum/common/utils.py b/magnum/common/utils.py index 7c8910504b..4bfc504c9a 100644 --- a/magnum/common/utils.py +++ b/magnum/common/utils.py @@ -549,8 +549,18 @@ def raise_exception_invalid_scheme(url): raise exception.Urllib2InvalidScheme(url=url) -def get_memory_bytes(memory): - """Kubernetes memory format must be in the format of: +def get_k8s_quantity(quantity): + """This function is used to get k8s quantity. + + It supports to get CPU and Memory quantity: + + Kubernetes cpu format must be in the format of: + + 'm' + for example: + 500m = 0.5 core of cpu + + Kubernetes memory format must be in the format of: signedNumber = digits|digits.digits|digits.|.digits @@ -558,23 +568,29 @@ def get_memory_bytes(memory): or suffix = E|e digits = digit | digit digit = 0|1|2|3|4|5|6|7|8|9 + + :param name: String value of a quantity such as '500m', '1G' + :returns: Quantity number + :raises: exception.UnsupportedK8sQuantityFormat if the quantity string + is a unsupported value """ + signed_num_regex = r"(^\d+\.\d+)|(^\d+\.)|(\.\d+)|(^\d+)" - matched_signed_number = re.search(signed_num_regex, memory) + matched_signed_number = re.search(signed_num_regex, quantity) if matched_signed_number is None: - raise exception.UnsupportedK8sMemoryFormat() + raise exception.UnsupportedK8sQuantityFormat() else: signed_number = matched_signed_number.group(0) - suffix = memory.replace(signed_number, '', 1) + suffix = quantity.replace(signed_number, '', 1) if suffix == '': - return float(memory) + return float(quantity) if re.search(r"^(Ki|Mi|Gi|Ti|Pi|Ei|m|k|M|G|T|P|E|'')$", suffix): return float(signed_number) * MEMORY_UNITS[suffix] elif re.search(r"^[E|e][+|-]?(\d+\.\d+$)|(\d+\.$)|(\.\d+$)|(\d+$)", suffix): return float(signed_number) * (10 ** float(suffix[1:])) else: - raise exception.UnsupportedK8sMemoryFormat() + raise exception.UnsupportedK8sQuantityFormat() def generate_password(length, symbolgroups=None): diff --git a/magnum/conductor/k8s_monitor.py b/magnum/conductor/k8s_monitor.py index 0db891f9a1..09ca1e6dda 100644 --- a/magnum/conductor/k8s_monitor.py +++ b/magnum/conductor/k8s_monitor.py @@ -32,6 +32,10 @@ class K8sMonitor(MonitorBase): 'unit': '%', 'func': 'compute_memory_util', }, + 'cpu_util': { + 'unit': '%', + 'func': 'compute_cpu_util', + }, } def pull_data(self): @@ -41,22 +45,28 @@ class K8sMonitor(MonitorBase): pods = k8s_api.list_namespaced_pod('default') self.data['pods'] = self._parse_pod_info(pods) - def compute_memory_util(self): - mem_total = 0 + def _compute_res_util(self, res): + res_total = 0 for node in self.data['nodes']: - mem_total += node['Memory'] - mem_reserved = 0 + res_total += node[res] + res_reserved = 0 for pod in self.data['pods']: - mem_reserved += pod['Memory'] + res_reserved += pod[res] - if mem_total == 0: + if res_total == 0: return 0 else: - return mem_reserved * 100 / mem_total + return res_reserved * 100 / res_total + + def compute_memory_util(self): + return self._compute_res_util('Memory') + + def compute_cpu_util(self): + return self._compute_res_util('Cpu') def _parse_pod_info(self, pods): - """Parse pods and retrieve memory details about each pod + """Parse pods and retrieve memory and cpu details about each pod :param pods: The output of k8s_api.list_namespaced_pods() For example: @@ -73,7 +83,8 @@ class K8sMonitor(MonitorBase): 'containers': [{ 'image': 'nginx', 'resources': {'requests': None, - 'limits': "{u'memory': u'1280e3'}"}, + 'limits': "{u'cpu': u'500m', + u'memory': u'1280e3'}"}, }], }, 'api_version': None, @@ -86,8 +97,8 @@ class K8sMonitor(MonitorBase): V1PodList object :return: Memory size of each pod. Example: - [{'Memory': 1280000.0}, - {'Memory': 1280000.0}] + [{'Memory': 1280000.0, cpu: 0.5}, + {'Memory': 1280000.0, cpu: 0.5}] """ pods = pods.items parsed_containers = [] @@ -95,23 +106,27 @@ class K8sMonitor(MonitorBase): containers = pod.spec.containers for container in containers: memory = 0 + cpu = 0 resources = container.resources limits = resources.limits if limits is not None: # Output of resources.limits is string # for example: - # limits = "{'memory': '1000Ki'}" + # limits = "{cpu': '500m': 'memory': '1000Ki'}" limits = ast.literal_eval(limits) if limits.get('memory', ''): - memory = utils.get_memory_bytes(limits['memory']) + memory = utils.get_k8s_quantity(limits['memory']) + if limits.get('cpu', ''): + cpu = utils.get_k8s_quantity(limits['cpu']) container_dict = { - 'Memory': memory + 'Memory': memory, + 'Cpu': cpu, } parsed_containers.append(container_dict) return parsed_containers def _parse_node_info(self, nodes): - """Parse nodes to retrieve memory of each node + """Parse nodes to retrieve memory and cpu of each node :param nodes: The output of k8s_api.list_namespaced_node() For example: @@ -119,7 +134,8 @@ class K8sMonitor(MonitorBase): 'items': [{ 'status': { 'phase': None, - 'capacity': "{u'memory': u'2049852Ki'}", + 'capacity': "{u'cpu': u'1', + u'memory': u'2049852Ki'}", }, }, 'api_version': None, @@ -132,9 +148,9 @@ class K8sMonitor(MonitorBase): magnum.common.pythonk8sclient.swagger_client.models.v1_node_list. V1NodeList object - :return: Memory size of each node. Excample: - [{'Memory': 1024.0}, - {'Memory': 1024.0}] + :return: CPU core number and Memory size of each node. Example: + [{'cpu': 1, 'Memory': 1024.0}, + {'cpu': 1, 'Memory': 1024.0}] """ nodes = nodes.items @@ -142,9 +158,10 @@ class K8sMonitor(MonitorBase): for node in nodes: # Output of node.status.capacity is strong # for example: - # capacity = "{'memory': '1000Ki'}" + # capacity = "{'cpu': '1', 'memory': '1000Ki'}" capacity = ast.literal_eval(node.status.capacity) - memory = utils.get_memory_bytes(capacity['memory']) - parsed_nodes.append({'Memory': memory}) + memory = utils.get_k8s_quantity(capacity['memory']) + cpu = int(capacity['cpu']) + parsed_nodes.append({'Memory': memory, 'Cpu': cpu}) return parsed_nodes diff --git a/magnum/tests/unit/common/test_utils.py b/magnum/tests/unit/common/test_utils.py index 9fdc8f0c37..c93db889e3 100644 --- a/magnum/tests/unit/common/test_utils.py +++ b/magnum/tests/unit/common/test_utils.py @@ -123,14 +123,15 @@ class UtilsTestCase(base.TestCase): utils.convert_to_list_dict(['first', 'second'], 'fred')) - def test_get_memory_bytes(self): - self.assertEqual(1024000.0, utils.get_memory_bytes('1000Ki')) - self.assertEqual(0.001, utils.get_memory_bytes('1E-3')) - self.assertEqual(0.5, utils.get_memory_bytes('0.0005k')) - self.assertEqual(1300000.0, utils.get_memory_bytes('1.3E+6')) - self.assertEqual(1300000.0, utils.get_memory_bytes('1.3E6')) - self.assertRaises(exception.UnsupportedK8sMemoryFormat, - utils.get_memory_bytes, '1E1E') + def test_get_k8s_quantity(self): + self.assertEqual(1024000.0, utils.get_k8s_quantity('1000Ki')) + self.assertEqual(0.001, utils.get_k8s_quantity('1E-3')) + self.assertEqual(0.5, utils.get_k8s_quantity('0.0005k')) + self.assertEqual(0.5, utils.get_k8s_quantity('500m')) + self.assertEqual(1300000.0, utils.get_k8s_quantity('1.3E+6')) + self.assertEqual(1300000.0, utils.get_k8s_quantity('1.3E6')) + self.assertRaises(exception.UnsupportedK8sQuantityFormat, + utils.get_k8s_quantity, '1E1E') class ExecuteTestCase(base.TestCase): diff --git a/magnum/tests/unit/conductor/test_monitors.py b/magnum/tests/unit/conductor/test_monitors.py index d26cd68a9b..78f51b6694 100644 --- a/magnum/tests/unit/conductor/test_monitors.py +++ b/magnum/tests/unit/conductor/test_monitors.py @@ -154,7 +154,7 @@ class MonitorsTestCase(base.TestCase): mock_nodes = mock.MagicMock() mock_node = mock.MagicMock() mock_node.status = mock.MagicMock() - mock_node.status.capacity = "{'memory': '2000Ki'}" + mock_node.status.capacity = "{'memory': '2000Ki', 'cpu': '1'}" mock_nodes.items = [mock_node] mock_k8s_api.return_value.list_namespaced_node.return_value = ( mock_nodes) @@ -163,16 +163,16 @@ class MonitorsTestCase(base.TestCase): mock_pod.spec = mock.MagicMock() mock_container = mock.MagicMock() mock_container.resources = mock.MagicMock() - mock_container.resources.limits = "{'memory':'100Mi'}" + mock_container.resources.limits = "{'memory': '100Mi', 'cpu': '500m'}" mock_pod.spec.containers = [mock_container] mock_pods.items = [mock_pod] mock_k8s_api.return_value.list_namespaced_pod.return_value = mock_pods self.k8s_monitor.pull_data() self.assertEqual(self.k8s_monitor.data['nodes'], - [{'Memory': 2048000.0}]) + [{'Memory': 2048000.0, 'Cpu': 1}]) self.assertEqual(self.k8s_monitor.data['pods'], - [{'Memory': 104857600.0}]) + [{'Memory': 104857600.0, 'Cpu': 0.5}]) def test_k8s_monitor_get_metric_names(self): k8s_metric_spec = 'magnum.conductor.k8s_monitor.K8sMonitor.'\ @@ -217,6 +217,31 @@ class MonitorsTestCase(base.TestCase): mem_util = self.k8s_monitor.compute_memory_util() self.assertEqual(0, mem_util) + def test_k8s_monitor_compute_cpu_util(self): + test_data = { + 'nodes': [ + { + 'Cpu': 1, + }, + ], + 'pods': [ + { + 'Cpu': 0.5, + }, + ], + } + self.k8s_monitor.data = test_data + cpu_util = self.k8s_monitor.compute_cpu_util() + self.assertEqual(50, cpu_util) + + test_data = { + 'nodes': [], + 'pods': [], + } + self.k8s_monitor.data = test_data + cpu_util = self.k8s_monitor.compute_cpu_util() + self.assertEqual(0, cpu_util) + def _test_mesos_monitor_pull_data( self, mock_url_get, state_json, expected_mem_total, expected_mem_used, expected_cpu_total, expected_cpu_used):