Add vendor methods for Intel Node Manager statistics

Intel Node Manager is able to provide some statistical parameters
directly without tools like ceilometer.
This patch adds vendor methods for statistics.

Change-Id: I9becb7e4f62c773d2a21fd53255acad3098b47a3

ironic_staging_drivers/common/exception.py

@@ -33,3 +33,7 @@ class AMTFailure(exception.IronicException):
 
 class LibvirtError(exception.IronicException):
     message = _("Libvirt call failed: %(err)s.")
+
+
+class InvalidIPMITimestamp(exception.IronicException):
+    pass

ironic_staging_drivers/intel_nm/__init__.py

@@ -52,7 +52,9 @@ class AgentAndIPMIToolIntelNMDriver(base.BaseDriver):
                         'get_nm_policy_suspend': self.nm_vendor,
                         'remove_nm_policy_suspend': self.nm_vendor,
                         'get_nm_capabilities': self.nm_vendor,
-                        'get_nm_version': self.nm_vendor}
+                        'get_nm_version': self.nm_vendor,
+                        'get_nm_statistics': self.nm_vendor,
+                        'reset_nm_statistics': self.nm_vendor}
         self.driver_passthru_mapping = {'lookup': self.agent_vendor}
         self.vendor = utils.MixinVendorInterface(
             self.mapping,

ironic_staging_drivers/intel_nm/nm_commands.py

@@ -12,14 +12,20 @@
 
 import binascii
 import collections
+import datetime
 import struct
 
-from ironic.common import exception
+from ironic.common import exception as ironic_exception
+from oslo_log import log
 import six
 
+from ironic_staging_drivers.common import exception
 from ironic_staging_drivers.common.i18n import _
+from ironic_staging_drivers.common.i18n import _LW
 
 
+LOG = log.getLogger(__name__)
+
 INTEL_NM_DOMAINS = {
     'platform': 0x00,
     'cpu': 0x01,
@@ -84,6 +90,27 @@ IPMI_VERSIONS = {
     0x03: '3.0'
 }
 
+INTEL_NM_STATISTICS = {
+    'global': {
+        'power': 0x01,
+        'temperature': 0x02,
+        'throttling': 0x03,
+        'airflow': 0x04,
+        'airflow_temperature': 0x05,
+        'chassis_power': 0x06,
+        'unhandled_requests': 0x1B,
+        'response_time': 0x1C,
+        'cpu_throttling': 0x1D,  # deprecated
+        'memory_throttling': 0x1E,  # deprecated
+        'communication_failures': 0x1F
+    },
+    'policy': {
+        'power': 0x11,
+        'trigger': 0x12,
+        'throttling': 0x13
+    }
+}
+
 
 def _reverse_dict(d):
     return {v: k for k, v in d.items()}
@@ -110,6 +137,12 @@ INTEL_NM_SUSPEND_SET = '0xC5'
 INTEL_NM_SUSPEND_GET = '0xC6'
 INTEL_NM_CAPABILITIES_GET = '0xC9'
 INTEL_NM_VERSION_GET = '0xCA'
+INTEL_NM_STATISTICS_RESET = '0xC7'
+INTEL_NM_STATISTICS_GET = '0xC8'
+
+_INVALID_TIME = datetime.datetime.utcfromtimestamp(0).isoformat()
+_UNSPECIFIED_TIMESTAMP = 0xFFFFFFFF
+_INIT_TIMESTAMP_MAX = 0x20000000
 
 
 def _handle_parsing_error(func):
@@ -121,11 +154,11 @@ def _handle_parsing_error(func):
         try:
             return func(raw_data)
         except (IndexError, struct.error):
-            raise exception.IPMIFailure(msg % _('has wrong length.'))
+            raise ironic_exception.IPMIFailure(msg % _('has wrong length.'))
         except KeyError:
-            raise exception.IPMIFailure(msg % _('is corrupted.'))
+            raise ironic_exception.IPMIFailure(msg % _('is corrupted.'))
         except ValueError:
-            raise exception.IPMIFailure(msg % _('cannot be converted.'))
+            raise ironic_exception.IPMIFailure(msg % _('cannot be converted.'))
 
     return wrapper
 
@@ -189,6 +222,19 @@ def _days_parse(pattern):
     return [day for day in INTEL_NM_DAYS if pattern & INTEL_NM_DAYS[day]]
 
 
+def _ipmi_timestamp_to_isotime(timestamp):
+    """Convert IPMI timestamp to iso8601."""
+    if timestamp == _UNSPECIFIED_TIMESTAMP:
+        raise exception.InvalidIPMITimestamp(_('IPMI timestamp is invalid or '
+                                               'unspecified'))
+    if timestamp <= _INIT_TIMESTAMP_MAX:
+        raise exception.InvalidIPMITimestamp(_('IPMI initialization is not '
+                                               'completed, relative time is '
+                                               '%d second') % timestamp)
+
+    return datetime.datetime.utcfromtimestamp(timestamp).isoformat()
+
+
 def set_policy(policy):
     """Return hex data for policy set command."""
     # NM defaults
@@ -401,6 +447,72 @@ def parse_version(raw_data):
     return version
 
 
+def reset_statistics(data):
+    """Return hex data for reset statistics command."""
+    cmd = _create_command_head(INTEL_NM_STATISTICS_RESET)
+    global_scope = data['scope'] == 'global'
+    if 'parameter_name' in data:
+        # statistics parameter is set, get corresponding value
+        mode = INTEL_NM_STATISTICS['global'][data['parameter_name']]
+        # domain id should be always 0x00 for global reset by parameter name
+        data['domain_id'] = 'platform'
+    else:
+        mode = 0x00 if global_scope else 0x01
+    _append_to_command(cmd, _hex(mode))
+    if global_scope:
+        data['policy_id'] = 0x00  # will be ignored
+    _add_domain_policy_id(cmd, data)
+
+    return cmd
+
+
+def get_statistics(data):
+    """Return hex data for get statistics command."""
+    cmd = _create_command_head(INTEL_NM_STATISTICS_GET)
+    scope = data['scope']
+    _append_to_command(cmd, _hex(
+                       INTEL_NM_STATISTICS[scope][data['parameter_name']]))
+    if scope == 'global':
+        data['policy_id'] = 0x00  # will be ignored
+    # case for "special" Node Manager global parameters (Mode 0x1B - 0x1F)
+    if 'domain_id' not in data:
+        data['domain_id'] = 'platform'  # 0x00
+    _add_domain_policy_id(cmd, data)
+
+    return cmd
+
+
+@_handle_parsing_error
+def parse_statistics(raw_data):
+    """Parse statistics data."""
+    statistics = {}
+    raw_int = _raw_to_int(raw_data)
+
+    statistics_values = struct.unpack('<HHHHII', bytearray(
+                                      raw_int[3:19]))
+    statistics_names = ('current_value', 'minimum_value',
+                        'maximum_value', 'average_value',
+                        'timestamp', 'reporting_period')
+    _add_to_dict(statistics, statistics_values, statistics_names)
+    try:
+        isotime = _ipmi_timestamp_to_isotime(statistics['timestamp'])
+    except exception.InvalidIPMITimestamp as e:
+        # there is not "bad time" in standard, reset to start the epoch
+        statistics['timestamp'] = _INVALID_TIME
+        LOG.warning(_LW('Invalid timestamp in Node Nanager statistics '
+                        'data: %s'), six.text_type(e))
+    else:
+        statistics['timestamp'] = isotime
+
+    statistics['domain_id'] = INTEL_NM_DOMAINS_REV[raw_int[19] & 0x0F]
+    statistics['administrative_enabled'] = bool(raw_int[19] & 0x10)
+    statistics['operational_state'] = bool(raw_int[19] & 0x20)
+    statistics['measurement_state'] = bool(raw_int[19] & 0x40)
+    statistics['activation_state'] = bool(raw_int[19] & 0x80)
+
+    return statistics
+
+
 # Code below taken from Ceilometer
 # Copyright 2014 Intel Corporation.
 def parse_slave_and_channel(file_path):

ironic_staging_drivers/intel_nm/nm_vendor.py

@@ -35,7 +35,7 @@ CONF.import_opt('tempdir', 'ironic.common.utils')
 LOG = log.getLogger(__name__)
 
 SCHEMAS = ('control_schema', 'get_cap_schema', 'main_ids_schema',
-           'policy_schema', 'suspend_schema')
+           'policy_schema', 'suspend_schema', 'statistics_schema')
 
 
 def _command_to_string(cmd):
@@ -144,6 +144,65 @@ class IntelNMVendorPassthru(base.VendorInterface):
             with open(filename, 'r') as sf:
                 setattr(self, schema, json.load(sf))
 
+    def _validate_policy_methods(self, method, **kwargs):
+        if method in ('get_nm_policy', 'remove_nm_policy',
+                      'get_nm_policy_suspend', 'remove_nm_policy_suspend'):
+            jsonschema.validate(kwargs, self.main_ids_schema)
+
+        elif method == 'control_nm_policy':
+            jsonschema.validate(kwargs, self.control_schema)
+            if kwargs['scope'] != 'global' and 'domain_id' not in kwargs:
+                raise exception.MissingParameterValue(_('Missing "domain_id"'))
+            if kwargs['scope'] == 'policy' and 'policy_id' not in kwargs:
+                raise exception.MissingParameterValue(_('Missing "policy_id"'))
+
+        elif method == 'set_nm_policy':
+            jsonschema.validate(kwargs, self.policy_schema)
+            if kwargs['policy_trigger'] == 'boot':
+                if not isinstance(kwargs['target_limit'], dict):
+                    raise exception.InvalidParameterValue(_('Invalid boot '
+                                                            'policy'))
+
+        elif method == 'set_nm_policy_suspend':
+            jsonschema.validate(kwargs, self.suspend_schema)
+
+        elif method == 'get_nm_capabilities':
+            jsonschema.validate(kwargs, self.get_cap_schema)
+
+    def _validate_statistics_methods(self, method, **kwargs):
+        jsonschema.validate(kwargs, self.statistics_schema)
+
+        global_params = ('unhandled_requests', 'response_time',
+                         'cpu_throttling', 'memory_throttling',
+                         'communication_failures')
+
+        if kwargs['scope'] == 'policy' and 'policy_id' not in kwargs:
+                raise exception.MissingParameterValue(_('Missing "policy_id"'))
+
+        if kwargs.get('parameter_name') not in global_params:
+            if 'domain_id' not in kwargs:
+                raise exception.MissingParameterValue(_('Missing "domain_id"'))
+
+        if method == 'reset_nm_statistics':
+            if 'parameter_name' in kwargs:
+                if kwargs['parameter_name'] not in global_params:
+                    raise exception.InvalidParameterValue(
+                        _('Invalid parameter name for resetting statistic, '
+                          'individual reset is possible only for: %s') %
+                        ', '.join(global_params))
+
+        elif method == 'get_nm_statistics':
+            if 'parameter_name' not in kwargs:
+                raise exception.MissingParameterValue(
+                    _('Parameter name is mandatory for getting statistics'))
+            # valid parameters depend on scope
+            if (kwargs['parameter_name'] not in
+                nm_commands.INTEL_NM_STATISTICS[kwargs['scope']]):
+                    raise exception.InvalidParameterValue(
+                        _('Invalid parameter name %(param)% for scope '
+                          '%(scope)s') % {'param': kwargs['parameter_name'],
+                                          'scope': kwargs['scope']})
+
     def get_properties(self):
         """Returns the properties of the interface.."""
         return {}
@@ -162,35 +221,10 @@ class IntelNMVendorPassthru(base.VendorInterface):
         :raises: MissingParameterValue if parameters missing in supplied data.
         """
         try:
-            if method in ('get_nm_policy', 'remove_nm_policy',
-                          'get_nm_policy_suspend', 'remove_nm_policy_suspend'):
-                jsonschema.validate(kwargs, self.main_ids_schema)
-
-            elif method == 'control_nm_policy':
-                jsonschema.validate(kwargs, self.control_schema)
-                no_domain = _('Missing "domain_id"')
-                no_policy = _('Missing "policy_id"')
-                if kwargs['scope'] == 'domain' and not kwargs.get('domain_id'):
-                    raise exception.MissingParameterValue(no_domain)
-                if kwargs['scope'] == 'policy':
-                    if not kwargs.get('domain_id'):
-                        raise exception.MissingParameterValue(no_domain)
-                    if not kwargs.get('policy_id'):
-                        raise exception.MissingParameterValue(no_policy)
-
-            elif method == 'set_nm_policy':
-                jsonschema.validate(kwargs, self.policy_schema)
-                if kwargs['policy_trigger'] == 'boot':
-                    if not isinstance(kwargs['target_limit'], dict):
-                        raise exception.InvalidParameterValue(_('Invalid boot '
-                                                                'policy'))
-
-            elif method == 'set_nm_policy_suspend':
-                jsonschema.validate(kwargs, self.suspend_schema)
-
-            elif method == 'get_nm_capabilities':
-                jsonschema.validate(kwargs, self.get_cap_schema)
-
+            if 'statistics' in method:
+                self._validate_statistics_methods(method, **kwargs)
+            else:
+                self._validate_policy_methods(method, **kwargs)
         except json_schema_exc.ValidationError as e:
             raise exception.InvalidParameterValue(_('Input data validation '
                                                     'error: %s') % e)
@@ -293,3 +327,26 @@ class IntelNMVendorPassthru(base.VendorInterface):
         """
         return _execute_nm_command(task, kwargs, nm_commands.get_version,
                                    nm_commands.parse_version)
+
+    @base.passthru(['GET'], async=False)
+    def get_nm_statistics(self, task, **kwargs):
+        """Get Intel Node Manager statistics.
+
+        :param task: a TaskManager instance.
+        :param kwargs: data passed to method.
+        :raises: IPMIFailure on an error.
+        :returns: a dictionary containing statistics info.
+        """
+        return _execute_nm_command(task, kwargs,
+                                   nm_commands.get_statistics,
+                                   nm_commands.parse_statistics)
+
+    @base.passthru(['PUT'])
+    def reset_nm_statistics(self, task, **kwargs):
+        """Reset Intel Node Manager statistics.
+
+        :param task: a TaskManager instance.
+        :param kwargs: data passed to method.
+        :raises: IPMIFailure on an error.
+        """
+        _execute_nm_command(task, kwargs, nm_commands.reset_statistics)

ironic_staging_drivers/intel_nm/statistics_schema.json

@@ -0,0 +1,24 @@
+{
+    "title": "Intel Node Manager statistics schema",
+    "type": "object",
+    "properties": {
+        "scope": {
+            "type": "string",
+            "enum": ["global", "policy"]
+        },
+        "parameter_name": {
+            "type": "string"
+        },
+        "domain_id": {
+            "type": "string",
+            "enum": ["platform", "cpu", "memory", "io", "protection"]
+        },
+        "policy_id": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 255
+        }
+    },
+    "required": ["scope"],
+    "additionalProperties": false
+}

ironic_staging_drivers/tests/unit/intel_nm/test_commands.py

@@ -243,6 +243,71 @@ class IntelNMPoliciesCommandTestCase(base.TestCase):
         self.assertRaises(exception.IPMIFailure, commands.parse_version,
                           raw_data)
 
+    def test_reset_statistics_global(self):
+        data = {'scope': 'global', 'domain_id': 'platform'}
+        expected = ['0x2E', '0xC7', '0x57', '0x01', '0x00', '0x00', '0x00',
+                    '0x00']
+        result = commands.reset_statistics(data)
+        self.assertEqual(expected, result)
+
+    def test_reset_statistics_policy(self):
+        data = {'scope': 'policy', 'domain_id': 'platform', 'policy_id': 111}
+        expected = ['0x2E', '0xC7', '0x57', '0x01', '0x00', '0x01', '0x00',
+                    '0x6F']
+        result = commands.reset_statistics(data)
+        self.assertEqual(expected, result)
+
+    def test_reset_statistics_parameter(self):
+        data = {'scope': 'global', 'parameter_name': 'response_time'}
+        expected = ['0x2E', '0xC7', '0x57', '0x01', '0x00', '0x1C', '0x00',
+                    '0x00']
+        result = commands.reset_statistics(data)
+        self.assertEqual(expected, result)
+
+    def test_get_statistics_global(self):
+        data = {'scope': 'global', 'domain_id': 'platform',
+                'parameter_name': 'power'}
+        expected = ['0x2E', '0xC8', '0x57', '0x01', '0x00', '0x01', '0x00',
+                    '0x00']
+        result = commands.get_statistics(data)
+        self.assertEqual(expected, result)
+
+    def test_get_statistics_global_without_domain(self):
+        data = {'scope': 'global', 'parameter_name': 'response_time'}
+        expected = ['0x2E', '0xC8', '0x57', '0x01', '0x00', '0x1C', '0x00',
+                    '0x00']
+        result = commands.get_statistics(data)
+        self.assertEqual(expected, result)
+
+    def test_get_statistics_policy(self):
+        data = {'scope': 'policy', 'domain_id': 'platform', 'policy_id': 111,
+                'parameter_name': 'power'}
+        expected = ['0x2E', '0xC8', '0x57', '0x01', '0x00', '0x11', '0x00',
+                    '0x6F']
+        result = commands.get_statistics(data)
+        self.assertEqual(expected, result)
+
+    def test_parse_statistics(self):
+        raw_data = ['0x00', '0x00', '0x00', '0x80', '0x00', '0x20', '0x00',
+                    '0xF0', '0x00', '0x60', '0x00', '0x00', '0x01', '0x20',
+                    '0x40', '0x01', '0x01', '0x00', '0x00', '0xF0']
+        expected = {'activation_state': True, 'administrative_enabled': True,
+                    'average_value': 96, 'current_value': 128,
+                    'domain_id': 'platform', 'maximum_value': 240,
+                    'measurement_state': True, 'minimum_value': 32,
+                    'operational_state': True, 'reporting_period': 257,
+                    'timestamp': '2004-02-03T20:13:52'}
+
+        result = commands.parse_statistics(raw_data)
+        self.assertEqual(expected, result)
+
+    def test_parse_statistics_invalid_timestamp(self):
+        raw_data = ['0x00', '0x00', '0x00', '0x80', '0x00', '0x20', '0x00',
+                    '0xF0', '0x00', '0x60', '0x00', '0xFF', '0xFF', '0xFF',
+                    '0xFF', '0x01', '0x01', '0x00', '0x00', '0xF0']
+        result = commands.parse_statistics(raw_data)
+        self.assertEqual(commands._INVALID_TIME, result['timestamp'])
+
 
 class ParsingFromFileTestCase(base.TestCase):
 

ironic_staging_drivers/tests/unit/intel_nm/test_vendor.py

@@ -47,6 +47,9 @@ _GET_CAP = {'domain_id': 'platform', 'policy_trigger': 'none',
 
 _CONTROL = {'scope': 'global', 'enable': True}
 
+_STATISTICS = {'scope': 'global', 'domain_id': 'platform',
+               'parameter_name': 'response_time'}
+
 _VENDOR_METHODS_DATA = {'get_nm_policy': _MAIN_IDS,
                         'remove_nm_policy': _MAIN_IDS,
                         'get_nm_policy_suspend': _MAIN_IDS,
@@ -54,7 +57,9 @@ _VENDOR_METHODS_DATA = {'get_nm_policy': _MAIN_IDS,
                         'set_nm_policy': _POLICY,
                         'set_nm_policy_suspend': _SUSPEND,
                         'get_nm_capabilities': _GET_CAP,
-                        'control_nm_policy': _CONTROL}
+                        'control_nm_policy': _CONTROL,
+                        'get_nm_statistics': _STATISTICS,
+                        'reset_nm_statistics': _STATISTICS}
 
 
 class IntelNMPassthruTestCase(db_base.DbTestCase):
@@ -267,6 +272,48 @@ class IntelNMPassthruTestCase(db_base.DbTestCase):
                               task.driver.vendor.validate, task,
                               'set_nm_policy', 'fake', **data)
 
+    def test_validate_statistics_no_policy(self):
+        data = {'scope': 'policy', 'domain_id': 'platform'}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            self.assertRaises(exception.MissingParameterValue,
+                              task.driver.vendor.validate, task,
+                              'reset_nm_statistics', 'fake', **data)
+
+    def test_validate_statistics_no_domain(self):
+        data = {'scope': 'global', 'parameter_name': 'power'}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            self.assertRaises(exception.InvalidParameterValue,
+                              task.driver.vendor.validate, task,
+                              'get_nm_statistics', 'fake', **data)
+
+    def test_reset_statistics_invalid_parameter(self):
+        data = {'scope': 'global', 'domain_id': 'platform',
+                'parameter_name': 'power'}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            self.assertRaises(exception.InvalidParameterValue,
+                              task.driver.vendor.validate, task,
+                              'reset_nm_statistics', 'fake', **data)
+
+    def test_get_statistics_no_parameter(self):
+        data = {'scope': 'global', 'domain_id': 'platform'}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            self.assertRaises(exception.MissingParameterValue,
+                              task.driver.vendor.validate, task,
+                              'get_nm_statistics', 'fake', **data)
+
+    def test_get_statistics_invalid_parameter(self):
+        data = {'scope': 'policy', 'domain_id': 'platform', 'policy_id': 111,
+                'parameter_name': 'response_time'}
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            self.assertRaises(exception.InvalidParameterValue,
+                              task.driver.vendor.validate, task,
+                              'get_nm_statistics', 'fake', **data)
+
     @mock.patch.object(nm_vendor, '_execute_nm_command', spec_set=True,
                        autospec=True)
     def test_control_nm_policy(self, mock_exec):
@@ -352,3 +399,22 @@ class IntelNMPassthruTestCase(db_base.DbTestCase):
             mock_exec.assert_called_once_with(task, {},
                                               nm_commands.get_version,
                                               nm_commands.parse_version)
+
+    @mock.patch.object(nm_vendor, '_execute_nm_command', spec_set=True,
+                       autospec=True)
+    def test_get_nm_statistics(self, mock_exec):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.driver.vendor.get_nm_statistics(task)
+            mock_exec.assert_called_once_with(task, {},
+                                              nm_commands.get_statistics,
+                                              nm_commands.parse_statistics)
+
+    @mock.patch.object(nm_vendor, '_execute_nm_command', spec_set=True,
+                       autospec=True)
+    def test_reset_nm_statistics(self, mock_exec):
+        with task_manager.acquire(self.context, self.node.uuid,
+                                  shared=False) as task:
+            task.driver.vendor.reset_nm_statistics(task)
+            mock_exec.assert_called_once_with(task, {},
+                                              nm_commands.reset_statistics)

releasenotes/notes/add-nm-statistics-methods-ddb803a9cf497f93.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Added two vendor methods "get_nm_statistics" and "reset_nm_statistics"
+    to Intel Node Manager driver for Node Manager statistics feature support