diff --git a/README.rst b/README.rst index 0c28a00..3bbfb7c 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,10 @@ -=============================== +==================== python-oneviewclient -=============================== +==================== -Library to use OneView to provide nodes for Ironic +Library to use HPE OneView to provide nodes for Ironic -This library adds a layer of communication between Ironic and HP OneView and +This library adds a communication layer between Ironic and OneView and abstracts the version of OneView in place. * Free software: Apache license @@ -13,6 +13,69 @@ abstracts the version of OneView in place. * Bugs: http://bugs.launchpad.net/python-oneviewclient Features --------- +======== -* TODO +Audit logging +------------- + +``python-oneviewclient`` is capable of logging method calls to OneView for +auditing. Currently, data about request timing and method names, parameters and +return values, can be recorded to be used in the auditing process to discover +and better understand hotspots, bottlenecks and to measure how the user code +and OneView integration performs. + +Enabling audit logging +"""""""""""""""""""""" + +To enable audit logging, the user code has to set three parameters in the +constructor of the client object. namely: ``audit_enabled``, ``audit_map_file`` +and ``audit_output_file``. ``audit_map_file`` and ``audit_output_file`` must be +filled with the absolute path to the audit map file and the audit output file. + +The audit map file +"""""""""""""""""" + +The audit map file is composed of two sections, ``audit`` and ``cases``. In the +``audit`` section there should be a ``case`` option where one, and just one, of +the audit logging ``cases`` needs to be specified. The ``cases`` section needs +to be filled with a name for a case followed by the methods that the user wants +to audit logging. The methods that are allowed for the audit logging are those +decorated by ``@auditing.audit`` in ``python-oneviewclient``. + +See an example of an audit map file:: + + [audit] + + # Case to be audit logged from those declared in cases section. + + case = case_number_one + + [cases] + + # Possible auditable case name followed by the audit loggable + # methods' names. + + case_number_one = first_method,second_method,third_method + case_number_two = first_method,third_method,fifth_method + + +The audit output file +""""""""""""""""""""" + +The result of the audit logging process is a JSON formatted file that can be +used by auditors, operators and engineers to obtain valuable information about +performance impacts of using ``python-oneviewclient`` to access OneView, +and better understand possible hotspots and bottlenecks in the integration of +the user code and OneView. + +See an example of an audit output file:: + + { + "method": "get_node_power_state", + "client_instance_id": 140396067361488, + "initial_time": "2016-08-29T17:32:01.403420", + "end_time": "2016-08-29T17:32:01.439126", + "is_ironic_request": true, + "is_oneview_request": false, + "ret": "Off" + } diff --git a/oneview_client/auditing.py b/oneview_client/auditing.py new file mode 100644 index 0000000..37601d7 --- /dev/null +++ b/oneview_client/auditing.py @@ -0,0 +1,75 @@ +# Copyright 2016 Hewlett Packard Enterprise Development LP. +# Copyright 2016 Universidade Federal de Campina Grande +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import json +import requests +import sys + +from six.moves import configparser + + +def read_audit_map_file(audit_cases_file): + config = configparser.RawConfigParser() + config.read(audit_cases_file) + audit_case = config.get('audit', 'case') + audit_case_methods = config.get('cases', audit_case) + return audit_case_methods.split(',') + + +def audit(f): + def wrapper(self, *args, **kwargs): + method = f.__name__ + client_instance_id = id(self) + method_caller = sys._getframe(1).f_code.co_name + + initial_time = datetime.datetime.now().isoformat() + ret = f(self, *args, **kwargs) + end_time = datetime.datetime.now().isoformat() + + is_ironic_request = ( + not callable(getattr(self, method_caller, False)) or + method_caller == '__init__' + ) + is_oneview_request = isinstance(ret, requests.models.Response) + + if self.audit_enabled and (method in self.audit_case_methods): + _log(self, method, ret, initial_time, end_time, client_instance_id, + is_ironic_request, is_oneview_request) + + return ret + return wrapper + + +def _log(cls, method, ret, initial_time, end_time, client_instance_id, + is_ironic_request, is_oneview_request): + if not cls.audit_case_methods: + raise ValueError('Missing audit case methods.') + + if not cls.audit_output_file: + raise ValueError('Missing audit output file.') + + data = dict(initial_time=initial_time, + end_time=end_time, + method=method, + ret=str(ret), + client_instance_id=client_instance_id, + is_ironic_request=is_ironic_request, + is_oneview_request=is_oneview_request) + + with open(cls.audit_output_file, 'a') as output: + json.dump(data, output) + output.write('\n') diff --git a/oneview_client/client.py b/oneview_client/client.py index 486da0b..deea4eb 100644 --- a/oneview_client/client.py +++ b/oneview_client/client.py @@ -21,6 +21,7 @@ import time import requests import retrying +from oneview_client import auditing from oneview_client import exceptions from oneview_client import ilo_utils from oneview_client import managers @@ -51,7 +52,8 @@ class BaseClient(object): def __init__( self, manager_url, username, password, allow_insecure_connections=False, tls_cacert_file='', - max_polling_attempts=20 + max_polling_attempts=20, audit_enabled=False, + audit_map_file='', audit_output_file='' ): self.manager_url = manager_url self.username = username @@ -59,21 +61,33 @@ class BaseClient(object): self.allow_insecure_connections = allow_insecure_connections self.tls_cacert_file = tls_cacert_file self.max_polling_attempts = max_polling_attempts + self.audit_enabled = audit_enabled + self.audit_map_file = audit_map_file + self.audit_output_file = audit_output_file + self.audit_case_methods = [] if self.allow_insecure_connections: requests.packages.urllib3.disable_warnings( requests.packages.urllib3.exceptions.InsecureRequestWarning ) + if self.audit_enabled: + self.audit_case_methods = auditing.read_audit_map_file( + self.audit_map_file + ) + self.session_id = self.get_session() + @auditing.audit def verify_credentials(self): return self._authenticate() + @auditing.audit def get_session(self): response = self._authenticate() return response.json().get('sessionID') + @auditing.audit def _authenticate(self): if self.manager_url in ("", None): raise exceptions.OneViewConnectionError( @@ -100,6 +114,7 @@ class BaseClient(object): else: return r + @auditing.audit def _logout(self): if self.manager_url in ("", None): raise exceptions.OneViewConnectionError( @@ -120,6 +135,7 @@ class BaseClient(object): if r.status_code == 400: raise exceptions.OneViewNotAuthorizedException() + @auditing.audit def _get_verify_connection_option(self): verify_status = False user_cacert = self.tls_cacert_file @@ -131,12 +147,14 @@ class BaseClient(object): verify_status = user_cacert return verify_status + @auditing.audit def verify_oneview_version(self): if not self._is_oneview_version_compatible(): msg = ("The version of the OneView's API is unsupported. " "Supported version is '%s'" % SUPPORTED_ONEVIEW_VERSION) raise exceptions.IncompatibleOneViewAPIVersion(msg) + @auditing.audit def _is_oneview_version_compatible(self): versions = self.get_oneview_version() v = SUPPORTED_ONEVIEW_VERSION @@ -144,6 +162,7 @@ class BaseClient(object): max_version_compatible = versions.get("currentVersion") >= v return min_version_compatible and max_version_compatible + @auditing.audit def get_oneview_version(self): url = '%s/rest/version' % self.manager_url headers = {"Accept-Language": "en_US"} @@ -153,7 +172,7 @@ class BaseClient(object): response = requests.get( url, headers=headers, verify=verify_ssl ) - _check_request_status(response) + self._check_request_status(response) versions = response.json() return versions @@ -188,12 +207,15 @@ class BaseClient(object): return json_response + @auditing.audit def _do_request(self, url, headers, body, request_type): verify_status = self._get_verify_connection_option() @retrying.retry( stop_max_attempt_number=self.max_polling_attempts, - retry_on_result=lambda response: _check_request_status(response), + retry_on_result=lambda response: self._check_request_status( + response + ), wait_fixed=WAIT_DO_REQUEST_IN_MILLISECONDS ) def request(url, headers, body, request_type): @@ -217,6 +239,7 @@ class BaseClient(object): return response return request(url, headers, body, request_type) + @auditing.audit def _wait_for_task_to_complete(self, task): @retrying.retry( retry_on_result=lambda task: task.get('percentComplete') < 100, @@ -242,6 +265,7 @@ class BaseClient(object): return task return wait(task) + @auditing.audit def _get_ilo_access(self, server_hardware_uuid): uri = ("/rest/server-hardware/%s/remoteConsoleUrl" % server_hardware_uuid) @@ -255,6 +279,7 @@ class BaseClient(object): return host_ip, token + @auditing.audit def get_sh_mac_from_ilo(self, server_hardware_uuid, nic_index=0): host_ip, ilo_token = self._get_ilo_access(server_hardware_uuid) try: @@ -262,6 +287,7 @@ class BaseClient(object): finally: ilo_utils.ilo_logout(host_ip, ilo_token) + @auditing.audit def _set_onetime_boot(self, server_hardware_uuid, boot_device): host_ip, ilo_token = self._get_ilo_access(server_hardware_uuid) oneview_ilo_mapping = { @@ -284,17 +310,44 @@ class BaseClient(object): finally: ilo_utils.ilo_logout(host_ip, ilo_token) + def _check_request_status(self, response): + repeat = False + status = response.status_code + + if status in (401, 403): + error_code = response.json().get('errorCode') + raise exceptions.OneViewNotAuthorizedException(error_code) + elif status == 404: + raise exceptions.OneViewResourceNotFoundError() + elif status in (408, 409,): + time.sleep(10) + repeat = True + elif status == 500: + raise exceptions.OneViewInternalServerError() + # Any other unexpected status are logged + elif status not in (200, 202,): + message = ( + "OneView appliance returned an unknown response status: %s" + % status + ) + raise exceptions.UnknowOneViewResponseError(message) + return repeat + class ClientV2(BaseClient): def __init__( self, manager_url, username, password, allow_insecure_connections=False, tls_cacert_file='', - max_polling_attempts=20 + max_polling_attempts=20, audit_enabled=False, + audit_map_file='', audit_output_file='' ): super(ClientV2, self).__init__(manager_url, username, password, allow_insecure_connections, - tls_cacert_file, max_polling_attempts) + tls_cacert_file, max_polling_attempts, + audit_enabled, audit_map_file, + audit_output_file) + # Next generation self.enclosure = managers.EnclosureManager(self) self.enclosure_group = managers.EnclosureGroupManager(self) @@ -312,11 +365,14 @@ class Client(BaseClient): def __init__( self, manager_url, username, password, allow_insecure_connections=False, tls_cacert_file='', - max_polling_attempts=20 + max_polling_attempts=20, audit_enabled=False, + audit_map_file='', audit_output_file='' ): super(Client, self).__init__(manager_url, username, password, allow_insecure_connections, - tls_cacert_file, max_polling_attempts) + tls_cacert_file, max_polling_attempts, + audit_enabled, audit_map_file, + audit_output_file) # Next generation self._enclosure_group = managers.EnclosureGroupManager(self) self._server_hardware = managers.ServerHardwareManager(self) @@ -326,22 +382,21 @@ class Client(BaseClient): self._server_profile = managers.ServerProfileManager(self) # --- Power Driver --- + @auditing.audit def get_node_power_state(self, node_info): return self.get_server_hardware(node_info).power_state + @auditing.audit def power_on(self, node_info): - if self.get_node_power_state(node_info) == \ - states.ONEVIEW_POWER_ON: + if self.get_node_power_state(node_info) == states.ONEVIEW_POWER_ON: ret = states.ONEVIEW_POWER_ON else: - ret = self.set_node_power_state( - node_info, states.ONEVIEW_POWER_ON - ) + ret = self.set_node_power_state(node_info, states.ONEVIEW_POWER_ON) return ret + @auditing.audit def power_off(self, node_info): - if self.get_node_power_state(node_info) == \ - states.ONEVIEW_POWER_OFF: + if self.get_node_power_state(node_info) == states.ONEVIEW_POWER_OFF: ret = states.ONEVIEW_POWER_OFF else: ret = self.set_node_power_state( @@ -349,6 +404,7 @@ class Client(BaseClient): ) return ret + @auditing.audit def set_node_power_state( self, node_info, state, press_type=MOMENTARY_PRESS ): @@ -366,13 +422,16 @@ class Client(BaseClient): return state # --- Management Driver --- + @auditing.audit def get_server_hardware(self, node_info): uuid = node_info['server_hardware_uri'].split("/")[-1] return self._server_hardware.get(uuid) + @auditing.audit def get_server_hardware_by_uuid(self, uuid): return self._server_hardware.get(uuid) + @auditing.audit def get_server_profile_from_hardware(self, node_info): server_hardware = self.get_server_hardware(node_info) server_profile_uri = server_hardware.server_profile_uri @@ -388,22 +447,27 @@ class Client(BaseClient): server_profile_uuid = server_profile_uri.split("/")[-1] return self._server_profile.get(server_profile_uuid) + @auditing.audit def get_server_profile_template(self, node_info): uuid = node_info['server_profile_template_uri'].split("/")[-1] return self._server_profile_template.get(uuid) + @auditing.audit def get_server_profile_template_by_uuid(self, uuid): return self._server_profile_template.get(uuid) + @auditing.audit def get_server_profile_by_uuid(self, uuid): return self._server_profile.get(uuid) + @auditing.audit def get_boot_order(self, node_info): server_profile = self.get_server_profile_from_hardware( node_info ) return server_profile.boot.get("order") + @auditing.audit def set_boot_device(self, node_info, new_primary_boot_device, onetime=False): if new_primary_boot_device is None: @@ -427,6 +491,7 @@ class Client(BaseClient): self._persistent_set_boot_device(node_info, boot_order, new_primary_boot_device) + @auditing.audit def _persistent_set_boot_device(self, node_info, boot_order, new_primary_boot_device): @@ -457,6 +522,7 @@ class Client(BaseClient): raise exceptions.OneViewErrorSettingBootDevice(e.message) # ---- Deploy Driver ---- + @auditing.audit def clone_template_and_apply(self, server_profile_name, server_hardware_uuid, @@ -495,8 +561,9 @@ class Client(BaseClient): uri=generate_new_profile_uri ) - server_profile_from_template_json['serverHardwareUri'] = \ + server_profile_from_template_json['serverHardwareUri'] = ( server_hardware_uri + ) server_profile_from_template_json['name'] = server_profile_name server_profile_from_template_json['serverProfileTemplateUri'] = "" @@ -513,14 +580,16 @@ class Client(BaseClient): except exceptions.OneViewTaskError as e: raise exceptions.OneViewServerProfileAssignmentError(e.message) - server_profile_uri = complete_task.get('associatedResource')\ - .get('resourceUri') + server_profile_uri = ( + complete_task.get('associatedResource').get('resourceUri') + ) uuid = server_profile_uri.split("/")[-1] server_profile = self.get_server_profile_by_uuid(uuid) return server_profile + @auditing.audit def delete_server_profile(self, uuid): if not uuid: raise ValueError('Missing Server Profile uuid.') @@ -539,6 +608,7 @@ class Client(BaseClient): return complete_task.get('associatedResource').get('resourceUri') # ---- Node Validate ---- + @auditing.audit def validate_node_server_hardware( self, node_info, node_memorymb, node_cpus ): @@ -561,6 +631,7 @@ class Client(BaseClient): ) raise exceptions.OneViewInconsistentResource(message) + @auditing.audit def validate_node_server_hardware_type(self, node_info): node_sht_uri = node_info.get('server_hardware_type_uri') server_hardware = self.get_server_hardware(node_info) @@ -575,9 +646,11 @@ class Client(BaseClient): ) raise exceptions.OneViewInconsistentResource(message) + @auditing.audit def check_server_profile_is_applied(self, node_info): self.get_server_profile_from_hardware(node_info) + @auditing.audit def validate_node_enclosure_group(self, node_info): server_hardware = self.get_server_hardware(node_info) sh_enclosure_group_uri = server_hardware.enclosure_group_uri @@ -598,6 +671,7 @@ class Client(BaseClient): ) raise exceptions.OneViewInconsistentResource(message) + @auditing.audit def is_node_port_mac_compatible_with_server_profile( self, node_info, ports ): @@ -646,6 +720,7 @@ class Client(BaseClient): ) raise exceptions.OneViewInconsistentResource(message) + @auditing.audit def is_node_port_mac_compatible_with_server_hardware( self, node_info, ports ): @@ -672,12 +747,14 @@ class Client(BaseClient): ) raise exceptions.OneViewInconsistentResource(message) + @auditing.audit def validate_node_server_profile_template(self, node_info): node_spt_uri = node_info.get('server_profile_template_uri') server_profile_template = self.get_server_profile_template(node_info) - spt_server_hardware_type_uri = server_profile_template \ - .server_hardware_type_uri + spt_server_hardware_type_uri = ( + server_profile_template.server_hardware_type_uri + ) spt_enclosure_group_uri = server_profile_template.enclosure_group_uri server_hardware = self.get_server_hardware(node_info) @@ -702,6 +779,7 @@ class Client(BaseClient): ) raise exceptions.OneViewInconsistentResource(message) + @auditing.audit def validate_spt_boot_connections(self, uuid): server_profile_template = self.get_server_profile_template_by_uuid( uuid @@ -716,27 +794,3 @@ class Client(BaseClient): " template %s." % server_profile_template.uri ) raise exceptions.OneViewInconsistentResource(message) - - -def _check_request_status(response): - repeat = False - status = response.status_code - - if status in (401, 403): - error_code = response.json().get('errorCode') - raise exceptions.OneViewNotAuthorizedException(error_code) - elif status == 404: - raise exceptions.OneViewResourceNotFoundError() - elif status in (408, 409,): - time.sleep(10) - repeat = True - elif status == 500: - raise exceptions.OneViewInternalServerError() - # Any other unexpected status are logged - elif status not in (200, 202): - message = ( - "OneView appliance returned an unknown response status: %s" - % status - ) - raise exceptions.UnknowOneViewResponseError(message) - return repeat diff --git a/oneview_client/tests/unit/test_auditing.py b/oneview_client/tests/unit/test_auditing.py new file mode 100644 index 0000000..7ec7826 --- /dev/null +++ b/oneview_client/tests/unit/test_auditing.py @@ -0,0 +1,99 @@ +# Copyright 2016 Hewlett Packard Enterprise Development LP. +# Copyright 2016 Universidade Federal de Campina Grande +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import unittest + +from oneview_client import auditing +from oneview_client import client + +FAKE_AUDITING_METHODS = ['auditable_method'] + + +class OneViewClientAuditTestCase(unittest.TestCase): + @mock.patch.object(client.ClientV2, '_authenticate', autospec=True) + def setUp(self, mock__authenticate): + super(OneViewClientAuditTestCase, self).setUp() + self.mock_read_audit_map_file = mock.Mock( + return_value=FAKE_AUDITING_METHODS + ) + self.mock_log = mock.Mock() + auditing.read_audit_map_file = self.mock_read_audit_map_file + auditing._log = self.mock_log + + self.oneview_client = client.ClientV2( + manager_url='https://1.2.3.4', + username='username', + password='password', + audit_enabled=True, + audit_map_file='oneview_audit_map_file.conf', + audit_output_file='oneview_audit_output_file.json' + ) + + @mock.patch.object(client.ClientV2, '_authenticate', autospec=True) + def test_oneview_auditing_enabled(self, mock__authenticate): + self.mock_read_audit_map_file.reset_mock() + self.oneview_client = client.ClientV2( + manager_url='https://1.2.3.4', + username='username', + password='password', + audit_enabled=True, + audit_map_file='oneview_audit_map_file.conf', + audit_output_file='oneview_audit_output_file.json' + ) + + self.assertTrue(self.mock_read_audit_map_file.called) + + @mock.patch.object(client.ClientV2, '_authenticate', autospec=True) + def test_oneview_auditing_disabled(self, mock__authenticate): + self.mock_read_audit_map_file.reset_mock() + self.oneview_client = client.ClientV2( + manager_url='https://1.2.3.4', + username='username', + password='password', + audit_enabled=False, + audit_map_file='oneview_audit_map_file.conf', + audit_output_file='oneview_audit_output_file.json' + ) + + self.assertFalse(self.mock_read_audit_map_file.called) + + def test_oneview_auditing_mapped_method(self): + + class Client(object): + audit_enabled = True + audit_output_file = 'oneview_audit_output_file.json' + audit_case_methods = FAKE_AUDITING_METHODS + + @auditing.audit + def auditable_method(self): + pass + + Client().auditable_method() + self.assertTrue(self.mock_log.called) + + def test_oneview_auditing_not_mapped_method(self): + + class Client(object): + audit_enabled = True + audit_output_file = 'oneview_audit_output_file.json' + audit_case_methods = FAKE_AUDITING_METHODS + + @auditing.audit + def not_auditable_method(self): + pass + + Client().not_auditable_method() + self.assertFalse(self.mock_log.called)