x/python-oneviewclient
Code Issues Proposed changes

Add auditing instrumentation for python-oneviewclient

Browse Source 
This change is about adding the ability of python-oneviewclient to
register time, parameters values and return values for methods
calls that do requests to OneView appliance. The instrumentation
implemented by this patch provides the necessary support that will
be needed to measure the performance of the OneView driver for
ironic and other tools.

Change-Id: Iae7fe8ec559ccb2cbf8f93486eb4a1a674d7053a
Co-Authored-By: Hugo Nicodemos <nicodemos@lsd.ufcg.edu.br>
This commit is contained in:
Xavier 2016-07-04 12:26:41 -03:00 committed by Gabriel Bezerra
parent a67c6c96c9
commit b79ce29232
4 changed files with 340 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
README.rst
View File

@ -1,10 +1,10 @@
===============================
====================
python-oneviewclient
===============================
====================
Library to use OneView to provide nodes for Ironic
Library to use HPE OneView to provide nodes for Ironic
This library adds a layer of communication between Ironic and HP OneView and
This library adds a communication layer between Ironic and OneView and
abstracts the version of OneView in place.
* Free software: Apache license
@ -13,6 +13,69 @@ abstracts the version of OneView in place.
* Bugs: http://bugs.launchpad.net/python-oneviewclient
Features
--------
========
* TODO
Audit logging
-------------
``python-oneviewclient`` is capable of logging method calls to OneView for
auditing. Currently, data about request timing and method names, parameters and
return values, can be recorded to be used in the auditing process to discover
and better understand hotspots, bottlenecks and to measure how the user code
and OneView integration performs.
Enabling audit logging
""""""""""""""""""""""
To enable audit logging, the user code has to set three parameters in the
constructor of the client object. namely: ``audit_enabled``, ``audit_map_file``
and ``audit_output_file``. ``audit_map_file`` and ``audit_output_file`` must be
filled with the absolute path to the audit map file and the audit output file.
The audit map file
""""""""""""""""""
The audit map file is composed of two sections, ``audit`` and ``cases``. In the
``audit`` section there should be a ``case`` option where one, and just one, of
the audit logging ``cases`` needs to be specified. The ``cases`` section needs
to be filled with a name for a case followed by the methods that the user wants
to audit logging. The methods that are allowed for the audit logging are those
decorated by ``@auditing.audit`` in ``python-oneviewclient``.
See an example of an audit map file::
[audit]
# Case to be audit logged from those declared in cases section.
case = case_number_one
[cases]
# Possible auditable case name followed by the audit loggable
# methods' names.
case_number_one = first_method,second_method,third_method
case_number_two = first_method,third_method,fifth_method
The audit output file
"""""""""""""""""""""
The result of the audit logging process is a JSON formatted file that can be
used by auditors, operators and engineers to obtain valuable information about
performance impacts of using ``python-oneviewclient`` to access OneView,
and better understand possible hotspots and bottlenecks in the integration of
the user code and OneView.
See an example of an audit output file::
{
"method": "get_node_power_state",
"client_instance_id": 140396067361488,
"initial_time": "2016-08-29T17:32:01.403420",
"end_time": "2016-08-29T17:32:01.439126",
"is_ironic_request": true,
"is_oneview_request": false,
"ret": "Off"
}

75
oneview_client/auditing.py Normal file
View File

@ -0,0 +1,75 @@
# Copyright 2016 Hewlett Packard Enterprise Development LP.
# Copyright 2016 Universidade Federal de Campina Grande
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import json
import requests
import sys
from six.moves import configparser
def read_audit_map_file(audit_cases_file):
config = configparser.RawConfigParser()
config.read(audit_cases_file)
audit_case = config.get('audit', 'case')
audit_case_methods = config.get('cases', audit_case)
return audit_case_methods.split(',')
def audit(f):
def wrapper(self, *args, **kwargs):
method = f.__name__
client_instance_id = id(self)
method_caller = sys._getframe(1).f_code.co_name
initial_time = datetime.datetime.now().isoformat()
ret = f(self, *args, **kwargs)
end_time = datetime.datetime.now().isoformat()
is_ironic_request = (
not callable(getattr(self, method_caller, False)) or
method_caller == '__init__'
)
is_oneview_request = isinstance(ret, requests.models.Response)
if self.audit_enabled and (method in self.audit_case_methods):
_log(self, method, ret, initial_time, end_time, client_instance_id,
is_ironic_request, is_oneview_request)
return ret
return wrapper
def _log(cls, method, ret, initial_time, end_time, client_instance_id,
is_ironic_request, is_oneview_request):
if not cls.audit_case_methods:
raise ValueError('Missing audit case methods.')
if not cls.audit_output_file:
raise ValueError('Missing audit output file.')
data = dict(initial_time=initial_time,
end_time=end_time,
method=method,
ret=str(ret),
client_instance_id=client_instance_id,
is_ironic_request=is_ironic_request,
is_oneview_request=is_oneview_request)
with open(cls.audit_output_file, 'a') as output:
json.dump(data, output)
output.write('\n')

140
oneview_client/client.py
View File

@ -21,6 +21,7 @@ import time
import requests
import retrying
from oneview_client import auditing
from oneview_client import exceptions
from oneview_client import ilo_utils
from oneview_client import managers
@ -51,7 +52,8 @@ class BaseClient(object):
def __init__(
self, manager_url, username, password,
allow_insecure_connections=False, tls_cacert_file='',
max_polling_attempts=20
max_polling_attempts=20, audit_enabled=False,
audit_map_file='', audit_output_file=''
):
self.manager_url = manager_url
self.username = username
@ -59,21 +61,33 @@ class BaseClient(object):
self.allow_insecure_connections = allow_insecure_connections
self.tls_cacert_file = tls_cacert_file
self.max_polling_attempts = max_polling_attempts
self.audit_enabled = audit_enabled
self.audit_map_file = audit_map_file
self.audit_output_file = audit_output_file
self.audit_case_methods = []
if self.allow_insecure_connections:
requests.packages.urllib3.disable_warnings(
requests.packages.urllib3.exceptions.InsecureRequestWarning
)
if self.audit_enabled:
self.audit_case_methods = auditing.read_audit_map_file(
self.audit_map_file
)
self.session_id = self.get_session()
@auditing.audit
def verify_credentials(self):
return self._authenticate()
@auditing.audit
def get_session(self):
response = self._authenticate()
return response.json().get('sessionID')
@auditing.audit
def _authenticate(self):
if self.manager_url in ("", None):
raise exceptions.OneViewConnectionError(
@ -100,6 +114,7 @@ class BaseClient(object):
else:
return r
@auditing.audit
def _logout(self):
if self.manager_url in ("", None):
raise exceptions.OneViewConnectionError(
@ -120,6 +135,7 @@ class BaseClient(object):
if r.status_code == 400:
raise exceptions.OneViewNotAuthorizedException()
@auditing.audit
def _get_verify_connection_option(self):
verify_status = False
user_cacert = self.tls_cacert_file
@ -131,12 +147,14 @@ class BaseClient(object):
verify_status = user_cacert
return verify_status
@auditing.audit
def verify_oneview_version(self):
if not self._is_oneview_version_compatible():
msg = ("The version of the OneView's API is unsupported. "
"Supported version is '%s'" % SUPPORTED_ONEVIEW_VERSION)
raise exceptions.IncompatibleOneViewAPIVersion(msg)
@auditing.audit
def _is_oneview_version_compatible(self):
versions = self.get_oneview_version()
v = SUPPORTED_ONEVIEW_VERSION
@ -144,6 +162,7 @@ class BaseClient(object):
max_version_compatible = versions.get("currentVersion") >= v
return min_version_compatible and max_version_compatible
@auditing.audit
def get_oneview_version(self):
url = '%s/rest/version' % self.manager_url
headers = {"Accept-Language": "en_US"}
@ -153,7 +172,7 @@ class BaseClient(object):
response = requests.get(
url, headers=headers, verify=verify_ssl
)
_check_request_status(response)
self._check_request_status(response)
versions = response.json()
return versions
@ -188,12 +207,15 @@ class BaseClient(object):
return json_response
@auditing.audit
def _do_request(self, url, headers, body, request_type):
verify_status = self._get_verify_connection_option()
@retrying.retry(
stop_max_attempt_number=self.max_polling_attempts,
retry_on_result=lambda response: _check_request_status(response),
retry_on_result=lambda response: self._check_request_status(
response
),
wait_fixed=WAIT_DO_REQUEST_IN_MILLISECONDS
)
def request(url, headers, body, request_type):
@ -217,6 +239,7 @@ class BaseClient(object):
return response
return request(url, headers, body, request_type)
@auditing.audit
def _wait_for_task_to_complete(self, task):
@retrying.retry(
retry_on_result=lambda task: task.get('percentComplete') < 100,
@ -242,6 +265,7 @@ class BaseClient(object):
return task
return wait(task)
@auditing.audit
def _get_ilo_access(self, server_hardware_uuid):
uri = ("/rest/server-hardware/%s/remoteConsoleUrl"
% server_hardware_uuid)
@ -255,6 +279,7 @@ class BaseClient(object):
return host_ip, token
@auditing.audit
def get_sh_mac_from_ilo(self, server_hardware_uuid, nic_index=0):
host_ip, ilo_token = self._get_ilo_access(server_hardware_uuid)
try:
@ -262,6 +287,7 @@ class BaseClient(object):
finally:
ilo_utils.ilo_logout(host_ip, ilo_token)
@auditing.audit
def _set_onetime_boot(self, server_hardware_uuid, boot_device):
host_ip, ilo_token = self._get_ilo_access(server_hardware_uuid)
oneview_ilo_mapping = {
@ -284,17 +310,44 @@ class BaseClient(object):
finally:
ilo_utils.ilo_logout(host_ip, ilo_token)
def _check_request_status(self, response):
repeat = False
status = response.status_code
if status in (401, 403):
error_code = response.json().get('errorCode')
raise exceptions.OneViewNotAuthorizedException(error_code)
elif status == 404:
raise exceptions.OneViewResourceNotFoundError()
elif status in (408, 409,):
time.sleep(10)
repeat = True
elif status == 500:
raise exceptions.OneViewInternalServerError()
# Any other unexpected status are logged
elif status not in (200, 202,):
message = (
"OneView appliance returned an unknown response status: %s"
% status
)
raise exceptions.UnknowOneViewResponseError(message)
return repeat
class ClientV2(BaseClient):
def __init__(
self, manager_url, username, password,
allow_insecure_connections=False, tls_cacert_file='',
max_polling_attempts=20
max_polling_attempts=20, audit_enabled=False,
audit_map_file='', audit_output_file=''
):
super(ClientV2, self).__init__(manager_url, username, password,
allow_insecure_connections,
tls_cacert_file, max_polling_attempts)
tls_cacert_file, max_polling_attempts,
audit_enabled, audit_map_file,
audit_output_file)
# Next generation
self.enclosure = managers.EnclosureManager(self)
self.enclosure_group = managers.EnclosureGroupManager(self)
@ -312,11 +365,14 @@ class Client(BaseClient):
def __init__(
self, manager_url, username, password,
allow_insecure_connections=False, tls_cacert_file='',
max_polling_attempts=20
max_polling_attempts=20, audit_enabled=False,
audit_map_file='', audit_output_file=''
):
super(Client, self).__init__(manager_url, username, password,
allow_insecure_connections,
tls_cacert_file, max_polling_attempts)
tls_cacert_file, max_polling_attempts,
audit_enabled, audit_map_file,
audit_output_file)
# Next generation
self._enclosure_group = managers.EnclosureGroupManager(self)
self._server_hardware = managers.ServerHardwareManager(self)
@ -326,22 +382,21 @@ class Client(BaseClient):
self._server_profile = managers.ServerProfileManager(self)
# --- Power Driver ---
@auditing.audit
def get_node_power_state(self, node_info):
return self.get_server_hardware(node_info).power_state
@auditing.audit
def power_on(self, node_info):
if self.get_node_power_state(node_info) == \
states.ONEVIEW_POWER_ON:
if self.get_node_power_state(node_info) == states.ONEVIEW_POWER_ON:
ret = states.ONEVIEW_POWER_ON
else:
ret = self.set_node_power_state(
node_info, states.ONEVIEW_POWER_ON
)
ret = self.set_node_power_state(node_info, states.ONEVIEW_POWER_ON)
return ret
@auditing.audit
def power_off(self, node_info):
if self.get_node_power_state(node_info) == \
states.ONEVIEW_POWER_OFF:
if self.get_node_power_state(node_info) == states.ONEVIEW_POWER_OFF:
ret = states.ONEVIEW_POWER_OFF
else:
ret = self.set_node_power_state(
@ -349,6 +404,7 @@ class Client(BaseClient):
)
return ret
@auditing.audit
def set_node_power_state(
self, node_info, state, press_type=MOMENTARY_PRESS
):
@ -366,13 +422,16 @@ class Client(BaseClient):
return state
# --- Management Driver ---
@auditing.audit
def get_server_hardware(self, node_info):
uuid = node_info['server_hardware_uri'].split("/")[-1]
return self._server_hardware.get(uuid)
@auditing.audit
def get_server_hardware_by_uuid(self, uuid):
return self._server_hardware.get(uuid)
@auditing.audit
def get_server_profile_from_hardware(self, node_info):
server_hardware = self.get_server_hardware(node_info)
server_profile_uri = server_hardware.server_profile_uri
@ -388,22 +447,27 @@ class Client(BaseClient):
server_profile_uuid = server_profile_uri.split("/")[-1]
return self._server_profile.get(server_profile_uuid)
@auditing.audit
def get_server_profile_template(self, node_info):
uuid = node_info['server_profile_template_uri'].split("/")[-1]
return self._server_profile_template.get(uuid)
@auditing.audit
def get_server_profile_template_by_uuid(self, uuid):
return self._server_profile_template.get(uuid)
@auditing.audit
def get_server_profile_by_uuid(self, uuid):
return self._server_profile.get(uuid)
@auditing.audit
def get_boot_order(self, node_info):
server_profile = self.get_server_profile_from_hardware(
node_info
)
return server_profile.boot.get("order")
@auditing.audit
def set_boot_device(self, node_info, new_primary_boot_device,
onetime=False):
if new_primary_boot_device is None:
@ -427,6 +491,7 @@ class Client(BaseClient):
self._persistent_set_boot_device(node_info, boot_order,
new_primary_boot_device)
@auditing.audit
def _persistent_set_boot_device(self, node_info, boot_order,
new_primary_boot_device):
@ -457,6 +522,7 @@ class Client(BaseClient):
raise exceptions.OneViewErrorSettingBootDevice(e.message)
# ---- Deploy Driver ----
@auditing.audit
def clone_template_and_apply(self,
server_profile_name,
server_hardware_uuid,
@ -495,8 +561,9 @@ class Client(BaseClient):
uri=generate_new_profile_uri
)
server_profile_from_template_json['serverHardwareUri'] = \
server_profile_from_template_json['serverHardwareUri'] = (
server_hardware_uri
)
server_profile_from_template_json['name'] = server_profile_name
server_profile_from_template_json['serverProfileTemplateUri'] = ""
@ -513,14 +580,16 @@ class Client(BaseClient):
except exceptions.OneViewTaskError as e:
raise exceptions.OneViewServerProfileAssignmentError(e.message)
server_profile_uri = complete_task.get('associatedResource')\
.get('resourceUri')
server_profile_uri = (
complete_task.get('associatedResource').get('resourceUri')
)
uuid = server_profile_uri.split("/")[-1]
server_profile = self.get_server_profile_by_uuid(uuid)
return server_profile
@auditing.audit
def delete_server_profile(self, uuid):
if not uuid:
raise ValueError('Missing Server Profile uuid.')
@ -539,6 +608,7 @@ class Client(BaseClient):
return complete_task.get('associatedResource').get('resourceUri')
# ---- Node Validate ----
@auditing.audit
def validate_node_server_hardware(
self, node_info, node_memorymb, node_cpus
):
@ -561,6 +631,7 @@ class Client(BaseClient):
)
raise exceptions.OneViewInconsistentResource(message)
@auditing.audit
def validate_node_server_hardware_type(self, node_info):
node_sht_uri = node_info.get('server_hardware_type_uri')
server_hardware = self.get_server_hardware(node_info)
@ -575,9 +646,11 @@ class Client(BaseClient):
)
raise exceptions.OneViewInconsistentResource(message)
@auditing.audit
def check_server_profile_is_applied(self, node_info):
self.get_server_profile_from_hardware(node_info)
@auditing.audit
def validate_node_enclosure_group(self, node_info):
server_hardware = self.get_server_hardware(node_info)
sh_enclosure_group_uri = server_hardware.enclosure_group_uri
@ -598,6 +671,7 @@ class Client(BaseClient):
)
raise exceptions.OneViewInconsistentResource(message)
@auditing.audit
def is_node_port_mac_compatible_with_server_profile(
self, node_info, ports
):
@ -646,6 +720,7 @@ class Client(BaseClient):
)
raise exceptions.OneViewInconsistentResource(message)
@auditing.audit
def is_node_port_mac_compatible_with_server_hardware(
self, node_info, ports
):
@ -672,12 +747,14 @@ class Client(BaseClient):
)
raise exceptions.OneViewInconsistentResource(message)
@auditing.audit
def validate_node_server_profile_template(self, node_info):
node_spt_uri = node_info.get('server_profile_template_uri')
server_profile_template = self.get_server_profile_template(node_info)
spt_server_hardware_type_uri = server_profile_template \
.server_hardware_type_uri
spt_server_hardware_type_uri = (
server_profile_template.server_hardware_type_uri
)
spt_enclosure_group_uri = server_profile_template.enclosure_group_uri
server_hardware = self.get_server_hardware(node_info)
@ -702,6 +779,7 @@ class Client(BaseClient):
)
raise exceptions.OneViewInconsistentResource(message)
@auditing.audit
def validate_spt_boot_connections(self, uuid):
server_profile_template = self.get_server_profile_template_by_uuid(
uuid
@ -716,27 +794,3 @@ class Client(BaseClient):
" template %s." % server_profile_template.uri
)
raise exceptions.OneViewInconsistentResource(message)
def _check_request_status(response):
repeat = False
status = response.status_code
if status in (401, 403):
error_code = response.json().get('errorCode')
raise exceptions.OneViewNotAuthorizedException(error_code)
elif status == 404:
raise exceptions.OneViewResourceNotFoundError()
elif status in (408, 409,):
time.sleep(10)
repeat = True
elif status == 500:
raise exceptions.OneViewInternalServerError()
# Any other unexpected status are logged
elif status not in (200, 202):
message = (
"OneView appliance returned an unknown response status: %s"
% status
)
raise exceptions.UnknowOneViewResponseError(message)
return repeat

99
oneview_client/tests/unit/test_auditing.py Normal file
View File

@ -0,0 +1,99 @@
# Copyright 2016 Hewlett Packard Enterprise Development LP.
# Copyright 2016 Universidade Federal de Campina Grande
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import unittest
from oneview_client import auditing
from oneview_client import client
FAKE_AUDITING_METHODS = ['auditable_method']
class OneViewClientAuditTestCase(unittest.TestCase):
@mock.patch.object(client.ClientV2, '_authenticate', autospec=True)
def setUp(self, mock__authenticate):
super(OneViewClientAuditTestCase, self).setUp()
self.mock_read_audit_map_file = mock.Mock(
return_value=FAKE_AUDITING_METHODS
)
self.mock_log = mock.Mock()
auditing.read_audit_map_file = self.mock_read_audit_map_file
auditing._log = self.mock_log
self.oneview_client = client.ClientV2(
manager_url='https://1.2.3.4',
username='username',
password='password',
audit_enabled=True,
audit_map_file='oneview_audit_map_file.conf',
audit_output_file='oneview_audit_output_file.json'
)
@mock.patch.object(client.ClientV2, '_authenticate', autospec=True)
def test_oneview_auditing_enabled(self, mock__authenticate):
self.mock_read_audit_map_file.reset_mock()
self.oneview_client = client.ClientV2(
manager_url='https://1.2.3.4',
username='username',
password='password',
audit_enabled=True,
audit_map_file='oneview_audit_map_file.conf',
audit_output_file='oneview_audit_output_file.json'
)
self.assertTrue(self.mock_read_audit_map_file.called)
@mock.patch.object(client.ClientV2, '_authenticate', autospec=True)
def test_oneview_auditing_disabled(self, mock__authenticate):
self.mock_read_audit_map_file.reset_mock()
self.oneview_client = client.ClientV2(
manager_url='https://1.2.3.4',
username='username',
password='password',
audit_enabled=False,
audit_map_file='oneview_audit_map_file.conf',
audit_output_file='oneview_audit_output_file.json'
)
self.assertFalse(self.mock_read_audit_map_file.called)
def test_oneview_auditing_mapped_method(self):
class Client(object):
audit_enabled = True
audit_output_file = 'oneview_audit_output_file.json'
audit_case_methods = FAKE_AUDITING_METHODS
@auditing.audit
def auditable_method(self):
pass
Client().auditable_method()
self.assertTrue(self.mock_log.called)
def test_oneview_auditing_not_mapped_method(self):
class Client(object):
audit_enabled = True
audit_output_file = 'oneview_audit_output_file.json'
audit_case_methods = FAKE_AUDITING_METHODS
@auditing.audit
def not_auditable_method(self):
pass
Client().not_auditable_method()
self.assertFalse(self.mock_log.called)