From 48771e6bfd5d6f37111ac3b2b8bc2feb4a9ea7cd Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Thu, 8 Jan 2015 13:58:40 -0500 Subject: [PATCH] Move files out of the namespace package Move the public API out of oslo.vmware to oslo_vmware. Retain the ability to import from the old namespace package for backwards compatibility for this release cycle. bp/drop-namespace-packages Change-Id: I11cf038c3832a7357ed53363d8ccf143daddd2a2 --- openstack-common.conf | 2 +- oslo/vmware/__init__.py | 26 + oslo/vmware/api.py | 489 +------------- oslo/vmware/constants.py | 21 +- oslo/vmware/exceptions.py | 250 +------ oslo/vmware/image_transfer.py | 597 +---------------- oslo/vmware/objects/datacenter.py | 16 +- oslo/vmware/objects/datastore.py | 307 +-------- oslo/vmware/pbm.py | 189 +----- oslo/vmware/rw_handles.py | 621 +---------------- oslo/vmware/service.py | 346 +--------- oslo/vmware/vim.py | 39 +- oslo/vmware/vim_util.py | 475 +------------ .../vmware/common => oslo_vmware}/__init__.py | 0 {oslo/vmware => oslo_vmware}/_i18n.py | 0 oslo_vmware/api.py | 500 ++++++++++++++ oslo_vmware/common/__init__.py | 0 .../common/loopingcall.py | 2 +- oslo_vmware/constants.py | 32 + oslo_vmware/exceptions.py | 261 ++++++++ oslo_vmware/image_transfer.py | 608 +++++++++++++++++ oslo_vmware/objects/__init__.py | 0 oslo_vmware/objects/datacenter.py | 27 + oslo_vmware/objects/datastore.py | 318 +++++++++ oslo_vmware/pbm.py | 200 ++++++ oslo_vmware/rw_handles.py | 632 ++++++++++++++++++ oslo_vmware/service.py | 357 ++++++++++ oslo_vmware/tests/__init__.py | 11 + oslo_vmware/tests/base.py | 53 ++ oslo_vmware/tests/objects/__init__.py | 0 oslo_vmware/tests/objects/test_datacenter.py | 30 + oslo_vmware/tests/objects/test_datastore.py | 384 +++++++++++ oslo_vmware/tests/test_api.py | 549 +++++++++++++++ oslo_vmware/tests/test_image_transfer.py | 552 +++++++++++++++ oslo_vmware/tests/test_pbm.py | 173 +++++ oslo_vmware/tests/test_rw_handles.py | 302 +++++++++ oslo_vmware/tests/test_service.py | 446 ++++++++++++ oslo_vmware/tests/test_vim.py | 110 +++ oslo_vmware/tests/test_vim_util.py | 363 ++++++++++ oslo_vmware/vim.py | 50 ++ oslo_vmware/vim_util.py | 486 ++++++++++++++ .../wsdl/5.5/core-types.xsd | 0 .../wsdl/5.5/pbm-messagetypes.xsd | 0 .../wsdl/5.5/pbm-types.xsd | 0 .../vmware => oslo_vmware}/wsdl/5.5/pbm.wsdl | 0 .../wsdl/5.5/pbmService.wsdl | 0 setup.cfg | 1 + tests/objects/test_datastore.py | 3 +- tests/test_api.py | 27 +- tests/test_image_transfer.py | 81 +-- tests/test_pbm.py | 7 +- tests/test_rw_handles.py | 6 +- tests/test_service.py | 3 +- tests/test_vim.py | 2 +- tests/test_vim_util.py | 16 +- tox.ini | 3 +- 56 files changed, 6538 insertions(+), 3435 deletions(-) rename {oslo/vmware/common => oslo_vmware}/__init__.py (100%) rename {oslo/vmware => oslo_vmware}/_i18n.py (100%) create mode 100644 oslo_vmware/api.py create mode 100644 oslo_vmware/common/__init__.py rename {oslo/vmware => oslo_vmware}/common/loopingcall.py (99%) create mode 100644 oslo_vmware/constants.py create mode 100644 oslo_vmware/exceptions.py create mode 100644 oslo_vmware/image_transfer.py create mode 100644 oslo_vmware/objects/__init__.py create mode 100644 oslo_vmware/objects/datacenter.py create mode 100644 oslo_vmware/objects/datastore.py create mode 100644 oslo_vmware/pbm.py create mode 100644 oslo_vmware/rw_handles.py create mode 100644 oslo_vmware/service.py create mode 100644 oslo_vmware/tests/__init__.py create mode 100644 oslo_vmware/tests/base.py create mode 100644 oslo_vmware/tests/objects/__init__.py create mode 100644 oslo_vmware/tests/objects/test_datacenter.py create mode 100644 oslo_vmware/tests/objects/test_datastore.py create mode 100644 oslo_vmware/tests/test_api.py create mode 100644 oslo_vmware/tests/test_image_transfer.py create mode 100644 oslo_vmware/tests/test_pbm.py create mode 100644 oslo_vmware/tests/test_rw_handles.py create mode 100644 oslo_vmware/tests/test_service.py create mode 100644 oslo_vmware/tests/test_vim.py create mode 100644 oslo_vmware/tests/test_vim_util.py create mode 100644 oslo_vmware/vim.py create mode 100644 oslo_vmware/vim_util.py rename {oslo/vmware => oslo_vmware}/wsdl/5.5/core-types.xsd (100%) rename {oslo/vmware => oslo_vmware}/wsdl/5.5/pbm-messagetypes.xsd (100%) rename {oslo/vmware => oslo_vmware}/wsdl/5.5/pbm-types.xsd (100%) rename {oslo/vmware => oslo_vmware}/wsdl/5.5/pbm.wsdl (100%) rename {oslo/vmware => oslo_vmware}/wsdl/5.5/pbmService.wsdl (100%) diff --git a/openstack-common.conf b/openstack-common.conf index c9ab1922..bf14b610 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -3,4 +3,4 @@ script=tools/run_cross_tests.sh # The base module to hold the copy of openstack.common -base=oslo.vmware +base=oslo_vmware diff --git a/oslo/vmware/__init__.py b/oslo/vmware/__init__.py index e69de29b..73e54f3d 100644 --- a/oslo/vmware/__init__.py +++ b/oslo/vmware/__init__.py @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import warnings + + +def deprecated(): + new_name = __name__.replace('.', '_') + warnings.warn( + ('The oslo namespace package is deprecated. Please use %s instead.' % + new_name), + DeprecationWarning, + stacklevel=3, + ) + + +deprecated() diff --git a/oslo/vmware/api.py b/oslo/vmware/api.py index efe36d48..626c4c86 100644 --- a/oslo/vmware/api.py +++ b/oslo/vmware/api.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,488 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Session and API call management for VMware ESX/VC server. - -This module contains classes to invoke VIM APIs. It supports -automatic session re-establishment and retry of API invocations -in case of connection problems or server API call overload. -""" - -import logging - -import six - -from oslo.utils import excutils -from oslo.vmware._i18n import _, _LE, _LI, _LW -from oslo.vmware.common import loopingcall -from oslo.vmware import exceptions -from oslo.vmware import pbm -from oslo.vmware import vim -from oslo.vmware import vim_util - - -LOG = logging.getLogger(__name__) - - -def _trunc_id(session_id): - """Returns truncated session id which is suitable for logging.""" - if session_id is not None: - return session_id[-5:] - - -# TODO(vbala) Move this class to excutils.py. -class RetryDecorator(object): - """Decorator for retrying a function upon suggested exceptions. - - The decorated function is retried for the given number of times, and the - sleep time between the retries is incremented until max sleep time is - reached. If the max retry count is set to -1, then the decorated function - is invoked indefinitely until an exception is thrown, and the caught - exception is not in the list of suggested exceptions. - """ - - def __init__(self, max_retry_count=-1, inc_sleep_time=10, - max_sleep_time=60, exceptions=()): - """Configure the retry object using the input params. - - :param max_retry_count: maximum number of times the given function must - be retried when one of the input 'exceptions' - is caught. When set to -1, it will be retried - indefinitely until an exception is thrown - and the caught exception is not in param - exceptions. - :param inc_sleep_time: incremental time in seconds for sleep time - between retries - :param max_sleep_time: max sleep time in seconds beyond which the sleep - time will not be incremented using param - inc_sleep_time. On reaching this threshold, - max_sleep_time will be used as the sleep time. - :param exceptions: suggested exceptions for which the function must be - retried - """ - self._max_retry_count = max_retry_count - self._inc_sleep_time = inc_sleep_time - self._max_sleep_time = max_sleep_time - self._exceptions = exceptions - self._retry_count = 0 - self._sleep_time = 0 - - def __call__(self, f): - - def _func(*args, **kwargs): - func_name = f.__name__ - result = None - try: - if self._retry_count: - LOG.debug("Invoking %(func_name)s; retry count is " - "%(retry_count)d.", - {'func_name': func_name, - 'retry_count': self._retry_count}) - result = f(*args, **kwargs) - except self._exceptions: - with excutils.save_and_reraise_exception() as ctxt: - LOG.warn(_LW("Exception which is in the suggested list of " - "exceptions occurred while invoking function:" - " %s."), - func_name, - exc_info=True) - if (self._max_retry_count != -1 and - self._retry_count >= self._max_retry_count): - LOG.error(_LE("Cannot retry upon suggested exception " - "since retry count (%(retry_count)d) " - "reached max retry count " - "(%(max_retry_count)d)."), - {'retry_count': self._retry_count, - 'max_retry_count': self._max_retry_count}) - else: - ctxt.reraise = False - self._retry_count += 1 - self._sleep_time += self._inc_sleep_time - return self._sleep_time - raise loopingcall.LoopingCallDone(result) - - def func(*args, **kwargs): - loop = loopingcall.DynamicLoopingCall(_func, *args, **kwargs) - evt = loop.start(periodic_interval_max=self._max_sleep_time) - LOG.debug("Waiting for function %s to return.", f.__name__) - return evt.wait() - - return func - - -class VMwareAPISession(object): - """Setup a session with the server and handles all calls made to it. - - Example: - api_session = VMwareAPISession('10.1.2.3', 'administrator', - 'password', 10, 0.1, - create_session=False, port=443) - result = api_session.invoke_api(vim_util, 'get_objects', - api_session.vim, 'HostSystem', 100) - """ - - def __init__(self, host, server_username, server_password, - api_retry_count, task_poll_interval, scheme='https', - create_session=True, wsdl_loc=None, pbm_wsdl_loc=None, - port=443, cacert=None, insecure=True): - """Initializes the API session with given parameters. - - :param host: ESX/VC server IP address or host name - :param port: port for connection - :param server_username: username of ESX/VC server admin user - :param server_password: password for param server_username - :param api_retry_count: number of times an API must be retried upon - session/connection related errors - :param task_poll_interval: sleep time in seconds for polling an - on-going async task as part of the API call - :param scheme: protocol-- http or https - :param create_session: whether to setup a connection at the time of - instance creation - :param wsdl_loc: VIM API WSDL file location - :param pbm_wsdl_loc: PBM service WSDL file location - :param cacert: Specify a CA bundle file to use in verifying a - TLS (https) server certificate. - :param insecure: Verify HTTPS connections using system certificates, - used only if cacert is not specified - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException - """ - self._host = host - self._port = port - self._server_username = server_username - self._server_password = server_password - self._api_retry_count = api_retry_count - self._task_poll_interval = task_poll_interval - self._scheme = scheme - self._vim_wsdl_loc = wsdl_loc - self._pbm_wsdl_loc = pbm_wsdl_loc - self._session_id = None - self._session_username = None - self._vim = None - self._pbm = None - self._cacert = cacert - self._insecure = insecure - if create_session: - self._create_session() - - def pbm_wsdl_loc_set(self, pbm_wsdl_loc): - self._pbm_wsdl_loc = pbm_wsdl_loc - self._pbm = None - LOG.info(_LI('PBM WSDL updated to %s'), pbm_wsdl_loc) - - @property - def vim(self): - if not self._vim: - self._vim = vim.Vim(protocol=self._scheme, - host=self._host, - port=self._port, - wsdl_url=self._vim_wsdl_loc, - cacert=self._cacert, - insecure=self._insecure) - return self._vim - - @property - def pbm(self): - if not self._pbm and self._pbm_wsdl_loc: - self._pbm = pbm.Pbm(protocol=self._scheme, - host=self._host, - port=self._port, - wsdl_url=self._pbm_wsdl_loc, - cacert=self._cacert, - insecure=self._insecure) - if self._session_id: - # To handle the case where pbm property is accessed after - # session creation. If pbm property is accessed before session - # creation, we set the cookie in _create_session. - self._pbm.set_soap_cookie(self._vim.get_http_cookie()) - return self._pbm - - @RetryDecorator(exceptions=(exceptions.VimConnectionException,)) - def _create_session(self): - """Establish session with the server.""" - session_manager = self.vim.service_content.sessionManager - # Login and create new session with the server for making API calls. - LOG.debug("Logging in with username = %s.", self._server_username) - session = self.vim.Login(session_manager, - userName=self._server_username, - password=self._server_password) - prev_session_id, self._session_id = self._session_id, session.key - # We need to save the username in the session since we may need it - # later to check active session. The SessionIsActive method requires - # the username parameter to be exactly same as that in the session - # object. We can't use the username used for login since the Login - # method ignores the case. - self._session_username = session.userName - LOG.info(_LI("Successfully established new session; session ID is " - "%s."), - _trunc_id(self._session_id)) - - # Terminate the previous session (if exists) for preserving sessions - # as there is a limit on the number of sessions we can have. - if prev_session_id: - try: - LOG.info(_LI("Terminating the previous session with ID = %s"), - _trunc_id(prev_session_id)) - self.vim.TerminateSession(session_manager, - sessionId=[prev_session_id]) - except Exception: - # This exception is something we can live with. It is - # just an extra caution on our side. The session might - # have been cleared already. We could have made a call to - # SessionIsActive, but that is an overhead because we - # anyway would have to call TerminateSession. - LOG.warn(_LW("Error occurred while terminating the previous " - "session with ID = %s."), - _trunc_id(prev_session_id), - exc_info=True) - - # Set PBM client cookie. - if self._pbm is not None: - self._pbm.set_soap_cookie(self._vim.get_http_cookie()) - - def logout(self): - """Log out and terminate the current session.""" - if self._session_id: - LOG.info(_LI("Logging out and terminating the current session " - "with ID = %s."), - _trunc_id(self._session_id)) - try: - self.vim.Logout(self.vim.service_content.sessionManager) - self._session_id = None - except Exception: - LOG.exception(_LE("Error occurred while logging out and " - "terminating the current session with " - "ID = %s."), - _trunc_id(self._session_id)) - else: - LOG.debug("No session exists to log out.") - - def invoke_api(self, module, method, *args, **kwargs): - """Wrapper method for invoking APIs. - - The API call is retried in the event of exceptions due to session - overload or connection problems. - - :param module: module corresponding to the VIM API call - :param method: method in the module which corresponds to the - VIM API call - :param args: arguments to the method - :param kwargs: keyword arguments to the method - :returns: response from the API call - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - - @RetryDecorator(max_retry_count=self._api_retry_count, - exceptions=(exceptions.VimSessionOverLoadException, - exceptions.VimConnectionException)) - def _invoke_api(module, method, *args, **kwargs): - try: - api_method = getattr(module, method) - return api_method(*args, **kwargs) - except exceptions.VimFaultException as excep: - # If this is due to an inactive session, we should re-create - # the session and retry. - if exceptions.NOT_AUTHENTICATED in excep.fault_list: - # The NotAuthenticated fault is set by the fault checker - # due to an empty response. An empty response could be a - # valid response; for e.g., response for the query to - # return the VMs in an ESX server which has no VMs in it. - # Also, the server responds with an empty response in the - # case of an inactive session. Therefore, we need a way to - # differentiate between these two cases. - if self.is_current_session_active(): - LOG.debug("Returning empty response for " - "%(module)s.%(method)s invocation.", - {'module': module, - 'method': method}) - return [] - else: - # empty response is due to an inactive session - excep_msg = ( - _("Current session: %(session)s is inactive; " - "re-creating the session while invoking " - "method %(module)s.%(method)s.") % - {'session': _trunc_id(self._session_id), - 'module': module, - 'method': method}) - LOG.warn(excep_msg, exc_info=True) - self._create_session() - raise exceptions.VimConnectionException(excep_msg, - excep) - else: - # no need to retry for other VIM faults like - # InvalidArgument - # Raise specific exceptions here if possible - if excep.fault_list: - LOG.debug("Fault list: %s", excep.fault_list) - fault = excep.fault_list[0] - clazz = exceptions.get_fault_class(fault) - raise clazz(six.text_type(excep), excep.details) - raise - - except exceptions.VimConnectionException: - with excutils.save_and_reraise_exception(): - # Re-create the session during connection exception only - # if the session has expired. Otherwise, it could be - # a transient issue. - if not self.is_current_session_active(): - LOG.warn(_LW("Re-creating session due to connection " - "problems while invoking method " - "%(module)s.%(method)s."), - {'module': module, - 'method': method}, - exc_info=True) - self._create_session() - - return _invoke_api(module, method, *args, **kwargs) - - def is_current_session_active(self): - """Check if current session is active. - - :returns: True if the session is active; False otherwise - """ - LOG.debug("Checking if the current session: %s is active.", - _trunc_id(self._session_id)) - - is_active = False - try: - is_active = self.vim.SessionIsActive( - self.vim.service_content.sessionManager, - sessionID=self._session_id, - userName=self._session_username) - except exceptions.VimException: - LOG.warn(_LW("Error occurred while checking whether the " - "current session: %s is active."), - _trunc_id(self._session_id), - exc_info=True) - - return is_active - - def wait_for_task(self, task): - """Waits for the given task to complete and returns the result. - - The task is polled until it is done. The method returns the task - information upon successful completion. In case of any error, - appropriate exception is raised. - - :param task: managed object reference of the task - :returns: task info upon successful completion of the task - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - loop = loopingcall.FixedIntervalLoopingCall(self._poll_task, task) - evt = loop.start(self._task_poll_interval) - LOG.debug("Waiting for the task: %s to complete.", task) - return evt.wait() - - def _poll_task(self, task): - """Poll the given task until completion. - - If the task completes successfully, the method returns the task info - using the input event (param done). In case of any error, appropriate - exception is set in the event. - - :param task: managed object reference of the task - """ - LOG.debug("Invoking VIM API to read info of task: %s.", task) - try: - task_info = self.invoke_api(vim_util, - 'get_object_property', - self.vim, - task, - 'info') - except exceptions.VimException: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE("Error occurred while reading info of " - "task: %s."), - task) - else: - if task_info.state in ['queued', 'running']: - if hasattr(task_info, 'progress'): - LOG.debug("Task: %(task)s progress is %(progress)s%%.", - {'task': task, - 'progress': task_info.progress}) - elif task_info.state == 'success': - LOG.debug("Task: %s status is success.", task) - raise loopingcall.LoopingCallDone(task_info) - else: - error_msg = six.text_type(task_info.error.localizedMessage) - error = task_info.error - name = error.fault.__class__.__name__ - task_ex = exceptions.get_fault_class(name)(error_msg) - raise task_ex - - def wait_for_lease_ready(self, lease): - """Waits for the given lease to be ready. - - This method return when the lease is ready. In case of any error, - appropriate exception is raised. - - :param lease: lease to be checked for - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - loop = loopingcall.FixedIntervalLoopingCall(self._poll_lease, lease) - evt = loop.start(self._task_poll_interval) - LOG.debug("Waiting for the lease: %s to be ready.", lease) - evt.wait() - - def _poll_lease(self, lease): - """Poll the state of the given lease. - - When the lease is ready, the event (param done) is notified. In case - of any error, appropriate exception is set in the event. - - :param lease: lease whose state is to be polled - """ - LOG.debug("Invoking VIM API to read state of lease: %s.", lease) - try: - state = self.invoke_api(vim_util, - 'get_object_property', - self.vim, - lease, - 'state') - except exceptions.VimException: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE("Error occurred while checking " - "state of lease: %s."), - lease) - else: - if state == 'ready': - LOG.debug("Lease: %s is ready.", lease) - raise loopingcall.LoopingCallDone() - elif state == 'initializing': - LOG.debug("Lease: %s is initializing.", lease) - elif state == 'error': - LOG.debug("Invoking VIM API to read lease: %s error.", - lease) - error_msg = self._get_error_message(lease) - excep_msg = _("Lease: %(lease)s is in error state. Details: " - "%(error_msg)s.") % {'lease': lease, - 'error_msg': error_msg} - LOG.error(excep_msg) - raise exceptions.VimException(excep_msg) - else: - # unknown state - excep_msg = _("Unknown state: %(state)s for lease: " - "%(lease)s.") % {'state': state, - 'lease': lease} - LOG.error(excep_msg) - raise exceptions.VimException(excep_msg) - - def _get_error_message(self, lease): - """Get error message associated with the given lease.""" - try: - return self.invoke_api(vim_util, - 'get_object_property', - self.vim, - lease, - 'error') - except exceptions.VimException: - LOG.warn(_LW("Error occurred while reading error message for " - "lease: %s."), - lease, - exc_info=True) - return "Unknown" +from oslo_vmware.api import * # noqa diff --git a/oslo/vmware/constants.py b/oslo/vmware/constants.py index d166d4e3..96b91dbd 100644 --- a/oslo/vmware/constants.py +++ b/oslo/vmware/constants.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,20 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. - -""" -Shared constants across the VMware ecosystem. -""" - -# Datacenter path for HTTP access to datastores if the target server is an ESX/ -# ESXi system: http://goo.gl/B5Htr8 for more information. -ESX_DATACENTER_PATH = 'ha-datacenter' - -# User Agent for HTTP requests between OpenStack and vCenter. -USER_AGENT = 'OpenStack-ESX-Adapter' - -# Key of the cookie header when using a SOAP session. -SOAP_COOKIE_KEY = 'vmware_soap_session' - -# Key of the cookie header when using a CGI session. -CGI_COOKIE_KEY = 'vmware_cgi_ticket' +from oslo_vmware.constants import * # noqa diff --git a/oslo/vmware/exceptions.py b/oslo/vmware/exceptions.py index 16fc677e..3c1453fd 100644 --- a/oslo/vmware/exceptions.py +++ b/oslo/vmware/exceptions.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,249 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Exception definitions. -""" - -import logging - -import six - -from oslo.vmware._i18n import _, _LE - -LOG = logging.getLogger(__name__) - -ALREADY_EXISTS = 'AlreadyExists' -CANNOT_DELETE_FILE = 'CannotDeleteFile' -FILE_ALREADY_EXISTS = 'FileAlreadyExists' -FILE_FAULT = 'FileFault' -FILE_LOCKED = 'FileLocked' -FILE_NOT_FOUND = 'FileNotFound' -INVALID_POWER_STATE = 'InvalidPowerState' -INVALID_PROPERTY = 'InvalidProperty' -NO_PERMISSION = 'NoPermission' -NOT_AUTHENTICATED = 'NotAuthenticated' -TASK_IN_PROGRESS = 'TaskInProgress' -DUPLICATE_NAME = 'DuplicateName' - - -class VimException(Exception): - """The base exception class for all exceptions this library raises.""" - - if six.PY2: - __str__ = lambda self: six.text_type(self).encode('utf8') - __unicode__ = lambda self: self.description - else: - __str__ = lambda self: self.description - - def __init__(self, message, cause=None): - Exception.__init__(self) - if isinstance(message, list): - # we need this to protect against developers using - # this method like VimFaultException - raise ValueError(_("exception_summary must not be a list")) - - self.msg = message - self.cause = cause - - @property - def description(self): - # NOTE(jecarey): self.msg and self.cause may be i18n objects - # that do not support str or concatenation, but can be used - # as replacement text. - descr = six.text_type(self.msg) - if self.cause: - descr += '\nCause: ' + six.text_type(self.cause) - return descr - - -class VimSessionOverLoadException(VimException): - """Thrown when there is an API call overload at the VMware server.""" - pass - - -class VimConnectionException(VimException): - """Thrown when there is a connection problem.""" - pass - - -class VimAttributeException(VimException): - """Thrown when a particular attribute cannot be found.""" - pass - - -class VimFaultException(VimException): - """Exception thrown when there are faults during VIM API calls.""" - - def __init__(self, fault_list, message, cause=None, details=None): - super(VimFaultException, self).__init__(message, cause) - if not isinstance(fault_list, list): - raise ValueError(_("fault_list must be a list")) - if details is not None and not isinstance(details, dict): - raise ValueError(_("details must be a dict")) - self.fault_list = fault_list - self.details = details - - if six.PY2: - __unicode__ = lambda self: self.description - else: - __str__ = lambda self: self.description - - @property - def description(self): - descr = VimException.description.fget(self) - if self.fault_list: - # fault_list doesn't contain non-ASCII chars, we can use str() - descr += '\nFaults: ' + str(self.fault_list) - if self.details: - # details may contain non-ASCII values - details = '{%s}' % ', '.join(["'%s': '%s'" % (k, v) for k, v in - six.iteritems(self.details)]) - descr += '\nDetails: ' + details - return descr - - -class ImageTransferException(VimException): - """Thrown when there is an error during image transfer.""" - pass - - -class VMwareDriverException(Exception): - """Base VMware Driver Exception - - To correctly use this class, inherit from it and define - a 'msg_fmt' property. That msg_fmt will get printf'd - with the keyword arguments provided to the constructor. - - """ - msg_fmt = _("An unknown exception occurred.") - - def __init__(self, message=None, details=None, **kwargs): - self.kwargs = kwargs - self.details = details - - if not message: - try: - message = self.msg_fmt % kwargs - - except Exception: - # kwargs doesn't match a variable in the message - # log the issue and the kwargs - LOG.exception(_LE('Exception in string format operation')) - for name, value in six.iteritems(kwargs): - LOG.error(_LE("%(name)s: %(value)s"), - {'name': name, 'value': value}) - # at least get the core message out if something happened - message = self.msg_fmt - - super(VMwareDriverException, self).__init__(message) - - -class VMwareDriverConfigurationException(VMwareDriverException): - """Base class for all configuration exceptions. - """ - msg_fmt = _("VMware Driver configuration fault.") - - -class UseLinkedCloneConfigurationFault(VMwareDriverConfigurationException): - msg_fmt = _("No default value for use_linked_clone found.") - - -class MissingParameter(VMwareDriverException): - msg_fmt = _("Missing parameter : %(param)s") - - -class AlreadyExistsException(VMwareDriverException): - msg_fmt = _("Resource already exists.") - code = 409 - - -class CannotDeleteFileException(VMwareDriverException): - msg_fmt = _("Cannot delete file.") - code = 403 - - -class FileAlreadyExistsException(VMwareDriverException): - msg_fmt = _("File already exists.") - code = 409 - - -class FileFaultException(VMwareDriverException): - msg_fmt = _("File fault.") - code = 409 - - -class FileLockedException(VMwareDriverException): - msg_fmt = _("File locked.") - code = 403 - - -class FileNotFoundException(VMwareDriverException): - msg_fmt = _("File not found.") - code = 404 - - -class InvalidPowerStateException(VMwareDriverException): - msg_fmt = _("Invalid power state.") - code = 409 - - -class InvalidPropertyException(VMwareDriverException): - msg_fmt = _("Invalid property.") - code = 400 - - -class NoPermissionException(VMwareDriverException): - msg_fmt = _("No Permission.") - code = 403 - - -class NotAuthenticatedException(VMwareDriverException): - msg_fmt = _("Not Authenticated.") - code = 403 - - -class TaskInProgress(VMwareDriverException): - msg_fmt = _("Entity has another operation in process.") - - -class DuplicateName(VMwareDriverException): - msg_fmt = _("Duplicate name.") - - -# Populate the fault registry with the exceptions that have -# special treatment. -_fault_classes_registry = { - ALREADY_EXISTS: AlreadyExistsException, - CANNOT_DELETE_FILE: CannotDeleteFileException, - FILE_ALREADY_EXISTS: FileAlreadyExistsException, - FILE_FAULT: FileFaultException, - FILE_LOCKED: FileLockedException, - FILE_NOT_FOUND: FileNotFoundException, - INVALID_POWER_STATE: InvalidPowerStateException, - INVALID_PROPERTY: InvalidPropertyException, - NO_PERMISSION: NoPermissionException, - NOT_AUTHENTICATED: NotAuthenticatedException, - TASK_IN_PROGRESS: TaskInProgress, - DUPLICATE_NAME: DuplicateName, -} - - -def get_fault_class(name): - """Get a named subclass of VMwareDriverException.""" - name = str(name) - fault_class = _fault_classes_registry.get(name) - if not fault_class: - LOG.debug('Fault %s not matched.', name) - fault_class = VMwareDriverException - return fault_class - - -def register_fault_class(name, exception): - fault_class = _fault_classes_registry.get(name) - if not issubclass(exception, VMwareDriverException): - raise TypeError(_("exception should be a subclass of " - "VMwareDriverException")) - if fault_class: - LOG.debug('Overriding exception for %s', name) - _fault_classes_registry[name] = exception +from oslo_vmware.exceptions import * # noqa diff --git a/oslo/vmware/image_transfer.py b/oslo/vmware/image_transfer.py index 90bec1f8..b49c4b15 100644 --- a/oslo/vmware/image_transfer.py +++ b/oslo/vmware/image_transfer.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,596 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Functions and classes for image transfer between ESX/VC & image service. -""" - -import errno -import logging - -from eventlet import event -from eventlet import greenthread -from eventlet import queue -from eventlet import timeout - -from oslo.vmware._i18n import _ -from oslo.vmware import constants -from oslo.vmware import exceptions -from oslo.vmware.objects import datastore as ds_obj -from oslo.vmware import rw_handles -from oslo.vmware import vim_util - - -LOG = logging.getLogger(__name__) - -IMAGE_SERVICE_POLL_INTERVAL = 5 -FILE_READ_WRITE_TASK_SLEEP_TIME = 0.01 -BLOCKING_QUEUE_SIZE = 10 - - -class BlockingQueue(queue.LightQueue): - """Producer-Consumer queue to share data between reader/writer threads.""" - - def __init__(self, max_size, max_transfer_size): - """Initializes the queue with the given parameters. - - :param max_size: maximum queue size; if max_size is less than zero or - None, the queue size is infinite. - :param max_transfer_size: maximum amount of data that can be - _transferred using this queue - """ - queue.LightQueue.__init__(self, max_size) - self._max_transfer_size = max_transfer_size - self._transferred = 0 - - def read(self, chunk_size): - """Read data from the queue. - - This method blocks until data is available. The input chunk size is - ignored since we have ensured that the data chunks written to the pipe - by the image reader thread is the same as the chunks asked for by the - image writer thread. - """ - if (self._max_transfer_size is 0 or - self._transferred < self._max_transfer_size): - data_item = self.get() - self._transferred += len(data_item) - return data_item - else: - LOG.debug("Completed transfer of size %s.", self._transferred) - return "" - - def write(self, data): - """Write data into the queue. - - :param data: data to be written - """ - self.put(data) - - # Below methods are provided in order to enable treating the queue - # as a file handle. - - def seek(self, offset, whence=0): - """Set the file's current position at the offset. - - This method throws IOError since seek cannot be supported for a pipe. - """ - raise IOError(errno.ESPIPE, "Illegal seek") - - def tell(self): - """Get the current file position.""" - return self._transferred - - def close(self): - pass - - def __str__(self): - return "blocking queue" - - -class ImageWriter(object): - """Class to write the image to the image service from an input file.""" - - def __init__(self, context, input_file, image_service, image_id, - image_meta=None): - """Initializes the image writer instance with given parameters. - - :param context: write context needed by the image service - :param input_file: file to read the image data from - :param image_service: handle to image service - :param image_id: ID of the image in the image service - :param image_meta: image meta-data - """ - if not image_meta: - image_meta = {} - - self._context = context - self._input_file = input_file - self._image_service = image_service - self._image_id = image_id - self._image_meta = image_meta - self._running = False - - def start(self): - """Start the image write task. - - :returns: the event indicating the status of the write task - """ - self._done = event.Event() - - def _inner(): - """Task performing the image write operation. - - This method performs image data transfer through an update call. - After the update, it waits until the image state becomes - 'active', 'killed' or unknown. If the final state is not 'active' - an instance of ImageTransferException is thrown. - - :raises: ImageTransferException - """ - LOG.debug("Calling image service update on image: %(image)s " - "with meta: %(meta)s", - {'image': self._image_id, - 'meta': self._image_meta}) - - try: - self._image_service.update(self._context, - self._image_id, - self._image_meta, - data=self._input_file) - self._running = True - while self._running: - LOG.debug("Retrieving status of image: %s.", - self._image_id) - image_meta = self._image_service.show(self._context, - self._image_id) - image_status = image_meta.get('status') - if image_status == 'active': - self.stop() - LOG.debug("Image: %s is now active.", - self._image_id) - self._done.send(True) - elif image_status == 'killed': - self.stop() - excep_msg = (_("Image: %s is in killed state.") % - self._image_id) - LOG.error(excep_msg) - excep = exceptions.ImageTransferException(excep_msg) - self._done.send_exception(excep) - elif image_status in ['saving', 'queued']: - LOG.debug("Image: %(image)s is in %(state)s state; " - "sleeping for %(sleep)d seconds.", - {'image': self._image_id, - 'state': image_status, - 'sleep': IMAGE_SERVICE_POLL_INTERVAL}) - greenthread.sleep(IMAGE_SERVICE_POLL_INTERVAL) - else: - self.stop() - excep_msg = (_("Image: %(image)s is in unknown " - "state: %(state)s.") % - {'image': self._image_id, - 'state': image_status}) - LOG.error(excep_msg) - excep = exceptions.ImageTransferException(excep_msg) - self._done.send_exception(excep) - except Exception as excep: - self.stop() - excep_msg = (_("Error occurred while writing image: %s") % - self._image_id) - LOG.exception(excep_msg) - excep = exceptions.ImageTransferException(excep_msg, excep) - self._done.send_exception(excep) - - LOG.debug("Starting image write task for image: %(image)s with" - " source: %(source)s.", - {'source': self._input_file, - 'image': self._image_id}) - greenthread.spawn(_inner) - return self._done - - def stop(self): - """Stop the image writing task.""" - LOG.debug("Stopping the writing task for image: %s.", - self._image_id) - self._running = False - - def wait(self): - """Wait for the image writer task to complete. - - This method returns True if the writer thread completes successfully. - In case of error, it raises ImageTransferException. - - :raises ImageTransferException - """ - return self._done.wait() - - def close(self): - """This is a NOP.""" - pass - - def __str__(self): - string = "Image Writer " % (self._input_file, - self._image_id) - return string - - -class FileReadWriteTask(object): - """Task which reads data from the input file and writes to the output file. - - This class defines the task which copies the given input file to the given - output file. The copy operation involves reading chunks of data from the - input file and writing the same to the output file. - """ - - def __init__(self, input_file, output_file): - """Initializes the read-write task with the given input parameters. - - :param input_file: the input file handle - :param output_file: the output file handle - """ - self._input_file = input_file - self._output_file = output_file - self._running = False - - def start(self): - """Start the file read - file write task. - - :returns: the event indicating the status of the read-write task - """ - self._done = event.Event() - - def _inner(): - """Task performing the file read-write operation.""" - self._running = True - while self._running: - try: - data = self._input_file.read(rw_handles.READ_CHUNKSIZE) - if not data: - LOG.debug("File read-write task is done.") - self.stop() - self._done.send(True) - self._output_file.write(data) - - # update lease progress if applicable - if hasattr(self._input_file, "update_progress"): - self._input_file.update_progress() - if hasattr(self._output_file, "update_progress"): - self._output_file.update_progress() - - greenthread.sleep(FILE_READ_WRITE_TASK_SLEEP_TIME) - except Exception as excep: - self.stop() - excep_msg = _("Error occurred during file read-write " - "task.") - LOG.exception(excep_msg) - excep = exceptions.ImageTransferException(excep_msg, excep) - self._done.send_exception(excep) - - LOG.debug("Starting file read-write task with source: %(source)s " - "and destination: %(dest)s.", - {'source': self._input_file, - 'dest': self._output_file}) - greenthread.spawn(_inner) - return self._done - - def stop(self): - """Stop the read-write task.""" - LOG.debug("Stopping the file read-write task.") - self._running = False - - def wait(self): - """Wait for the file read-write task to complete. - - This method returns True if the read-write thread completes - successfully. In case of error, it raises ImageTransferException. - - :raises: ImageTransferException - """ - return self._done.wait() - - def __str__(self): - string = ("File Read-Write Task " % - (self._input_file, self._output_file)) - return string - - -# Functions to perform image transfer between VMware servers and image service. - - -def _start_transfer(context, timeout_secs, read_file_handle, max_data_size, - write_file_handle=None, image_service=None, image_id=None, - image_meta=None): - """Start the image transfer. - - The image reader reads the data from the image source and writes to the - blocking queue. The image source is always a file handle (VmdkReadHandle - or ImageReadHandle); therefore, a FileReadWriteTask is created for this - transfer. The image writer reads the data from the blocking queue and - writes it to the image destination. The image destination is either a - file or VMDK in VMware datastore or an image in the image service. - - If the destination is a file or VMDK in VMware datastore, the method - creates a FileReadWriteTask which reads from the blocking queue and - writes to either FileWriteHandle or VmdkWriteHandle. In the case of - image service as the destination, an instance of ImageWriter task is - created which reads from the blocking queue and writes to the image - service. - - :param context: write context needed for the image service - :param timeout_secs: time in seconds to wait for the transfer to complete - :param read_file_handle: handle to read data from - :param max_data_size: maximum transfer size - :param write_file_handle: handle to write data to; if this is None, then - param image_service and param image_id should - be set. - :param image_service: image service handle - :param image_id: ID of the image in the image service - :param image_meta: image meta-data - :raises: ImageTransferException, ValueError - """ - - # Create the blocking queue - blocking_queue = BlockingQueue(BLOCKING_QUEUE_SIZE, max_data_size) - - # Create the image reader - reader = FileReadWriteTask(read_file_handle, blocking_queue) - - # Create the image writer - if write_file_handle: - # File or VMDK in VMware datastore is the image destination - writer = FileReadWriteTask(blocking_queue, write_file_handle) - elif image_service and image_id: - # Image service image is the destination - writer = ImageWriter(context, - blocking_queue, - image_service, - image_id, - image_meta) - else: - excep_msg = _("No image destination given.") - LOG.error(excep_msg) - raise ValueError(excep_msg) - - # Start the reader and writer - LOG.debug("Starting image transfer with reader: %(reader)s and writer: " - "%(writer)s", - {'reader': reader, - 'writer': writer}) - reader.start() - writer.start() - timer = timeout.Timeout(timeout_secs) - try: - # Wait for the reader and writer to complete - reader.wait() - writer.wait() - except (timeout.Timeout, exceptions.ImageTransferException) as excep: - excep_msg = (_("Error occurred during image transfer with reader: " - "%(reader)s and writer: %(writer)s") % - {'reader': reader, - 'writer': writer}) - LOG.exception(excep_msg) - reader.stop() - writer.stop() - - if isinstance(excep, exceptions.ImageTransferException): - raise - raise exceptions.ImageTransferException(excep_msg, excep) - finally: - timer.cancel() - read_file_handle.close() - if write_file_handle: - write_file_handle.close() - - -def download_image(image, image_meta, session, datastore, rel_path, - bypass=True, timeout_secs=7200): - """Transfer an image to a datastore. - - :param image: file-like iterator - :param image_meta: image metadata - :param session: VMwareAPISession object - :param datastore: Datastore object - :param rel_path: path where the file will be stored in the datastore - :param bypass: if set to True, bypass vCenter to download the image - :param timeout_secs: time in seconds to wait for the xfer to complete - """ - image_size = int(image_meta['size']) - method = 'PUT' - if bypass: - hosts = datastore.get_connected_hosts(session) - host = ds_obj.Datastore.choose_host(hosts) - host_name = session.invoke_api(vim_util, 'get_object_property', - session.vim, host, 'name') - ds_url = datastore.build_url(session._scheme, host_name, rel_path, - constants.ESX_DATACENTER_PATH) - cookie = ds_url.get_transfer_ticket(session, method) - conn = ds_url.connect(method, image_size, cookie) - else: - ds_url = datastore.build_url(session._scheme, session._host, rel_path) - cookie = '%s=%s' % (constants.SOAP_COOKIE_KEY, - session.vim.get_http_cookie().strip("\"")) - conn = ds_url.connect(method, image_size, cookie) - conn.write = conn.send - - read_handle = rw_handles.ImageReadHandle(image) - _start_transfer(None, timeout_secs, read_handle, image_size, - write_file_handle=conn) - - -def download_flat_image(context, timeout_secs, image_service, image_id, - **kwargs): - """Download flat image from the image service to VMware server. - - :param context: image service write context - :param timeout_secs: time in seconds to wait for the download to complete - :param image_service: image service handle - :param image_id: ID of the image to be downloaded - :param kwargs: keyword arguments to configure the destination - file write handle - :raises: VimConnectionException, ImageTransferException, ValueError - """ - LOG.debug("Downloading image: %s from image service as a flat file.", - image_id) - - # TODO(vbala) catch specific exceptions raised by download call - read_iter = image_service.download(context, image_id) - read_handle = rw_handles.ImageReadHandle(read_iter) - file_size = int(kwargs.get('image_size')) - write_handle = rw_handles.FileWriteHandle(kwargs.get('host'), - kwargs.get('port'), - kwargs.get('data_center_name'), - kwargs.get('datastore_name'), - kwargs.get('cookies'), - kwargs.get('file_path'), - file_size, - cacerts=kwargs.get('cacerts')) - _start_transfer(context, - timeout_secs, - read_handle, - file_size, - write_file_handle=write_handle) - LOG.debug("Downloaded image: %s from image service as a flat file.", - image_id) - - -def download_stream_optimized_data(context, timeout_secs, read_handle, - **kwargs): - """Download stream optimized data to VMware server. - - :param context: image service write context - :param timeout_secs: time in seconds to wait for the download to complete - :param read_handle: handle from which to read the image data - :param kwargs: keyword arguments to configure the destination - VMDK write handle - :returns: managed object reference of the VM created for import to VMware - server - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException, - ImageTransferException, ValueError - """ - file_size = int(kwargs.get('image_size')) - write_handle = rw_handles.VmdkWriteHandle(kwargs.get('session'), - kwargs.get('host'), - kwargs.get('port'), - kwargs.get('resource_pool'), - kwargs.get('vm_folder'), - kwargs.get('vm_import_spec'), - file_size) - _start_transfer(context, - timeout_secs, - read_handle, - file_size, - write_file_handle=write_handle) - return write_handle.get_imported_vm() - - -def download_stream_optimized_image(context, timeout_secs, image_service, - image_id, **kwargs): - """Download stream optimized image from image service to VMware server. - - :param context: image service write context - :param timeout_secs: time in seconds to wait for the download to complete - :param image_service: image service handle - :param image_id: ID of the image to be downloaded - :param kwargs: keyword arguments to configure the destination - VMDK write handle - :returns: managed object reference of the VM created for import to VMware - server - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException, - ImageTransferException, ValueError - """ - LOG.debug("Downloading image: %s from image service as a stream " - "optimized file.", - image_id) - - # TODO(vbala) catch specific exceptions raised by download call - read_iter = image_service.download(context, image_id) - read_handle = rw_handles.ImageReadHandle(read_iter) - imported_vm = download_stream_optimized_data(context, timeout_secs, - read_handle, **kwargs) - - LOG.debug("Downloaded image: %s from image service as a stream " - "optimized file.", - image_id) - return imported_vm - - -def copy_stream_optimized_disk( - context, timeout_secs, write_handle, **kwargs): - """Copy virtual disk from VMware server to the given write handle. - - :param context: context - :param timeout_secs: time in seconds to wait for the copy to complete - :param write_handle: copy destination - :param kwargs: keyword arguments to configure the source - VMDK read handle - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException, - ImageTransferException, ValueError - """ - vmdk_file_path = kwargs.get('vmdk_file_path') - LOG.debug("Copying virtual disk: %(vmdk_path)s to %(dest)s.", - {'vmdk_path': vmdk_file_path, - 'dest': write_handle.name}) - file_size = kwargs.get('vmdk_size') - read_handle = rw_handles.VmdkReadHandle(kwargs.get('session'), - kwargs.get('host'), - kwargs.get('port'), - kwargs.get('vm'), - kwargs.get('vmdk_file_path'), - file_size) - _start_transfer(context, timeout_secs, read_handle, file_size, - write_file_handle=write_handle) - LOG.debug("Downloaded virtual disk: %s.", vmdk_file_path) - - -def upload_image(context, timeout_secs, image_service, image_id, owner_id, - **kwargs): - """Upload the VM's disk file to image service. - - :param context: image service write context - :param timeout_secs: time in seconds to wait for the upload to complete - :param image_service: image service handle - :param image_id: upload destination image ID - :param kwargs: keyword arguments to configure the source - VMDK read handle - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException, - ImageTransferException, ValueError - """ - - LOG.debug("Uploading to image: %s.", image_id) - file_size = kwargs.get('vmdk_size') - read_handle = rw_handles.VmdkReadHandle(kwargs.get('session'), - kwargs.get('host'), - kwargs.get('port'), - kwargs.get('vm'), - kwargs.get('vmdk_file_path'), - file_size) - - # Set the image properties. It is important to set the 'size' to 0. - # Otherwise, the image service client will use the VM's disk capacity - # which will not be the image size after upload, since it is converted - # to a stream-optimized sparse disk. - image_metadata = {'disk_format': 'vmdk', - 'is_public': kwargs.get('is_public'), - 'name': kwargs.get('image_name'), - 'status': 'active', - 'container_format': 'bare', - 'size': 0, - 'properties': {'vmware_image_version': - kwargs.get('image_version'), - 'vmware_disktype': 'streamOptimized', - 'owner_id': owner_id}} - - # Passing 0 as the file size since data size to be transferred cannot be - # predetermined. - _start_transfer(context, - timeout_secs, - read_handle, - 0, - image_service=image_service, - image_id=image_id, - image_meta=image_metadata) - LOG.debug("Uploaded image: %s.", image_id) +from oslo_vmware.image_transfer import * # noqa diff --git a/oslo/vmware/objects/datacenter.py b/oslo/vmware/objects/datacenter.py index 0a54fcbc..597b90f6 100644 --- a/oslo/vmware/objects/datacenter.py +++ b/oslo/vmware/objects/datacenter.py @@ -1,5 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,16 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.vmware._i18n import _ - - -class Datacenter(object): - - def __init__(self, ref, name): - """Datacenter object holds ref and name together for convenience.""" - if name is None: - raise ValueError(_("Datacenter name cannot be None")) - if ref is None: - raise ValueError(_("Datacenter reference cannot be None")) - self.ref = ref - self.name = name +from oslo_vmware.objects.datacenter import * # noqa diff --git a/oslo/vmware/objects/datastore.py b/oslo/vmware/objects/datastore.py index abeb26a5..302ddbf4 100644 --- a/oslo/vmware/objects/datastore.py +++ b/oslo/vmware/objects/datastore.py @@ -1,5 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,307 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -import logging -import posixpath -import random - -import six.moves.http_client as httplib -import six.moves.urllib.parse as urlparse - -from oslo.vmware._i18n import _ -from oslo.vmware import constants -from oslo.vmware import exceptions -from oslo.vmware import vim_util - -LOG = logging.getLogger(__name__) - - -class Datastore(object): - - def __init__(self, ref, name, capacity=None, freespace=None, - type=None, datacenter=None): - """Datastore object holds ref and name together for convenience. - - :param ref: a vSphere reference to a datastore - :param name: vSphere unique name for this datastore - :param capacity: (optional) capacity in bytes of this datastore - :param freespace: (optional) free space in bytes of datastore - :param type: (optional) datastore type - :param datacenter: (optional) oslo.vmware Datacenter object - """ - if name is None: - raise ValueError(_("Datastore name cannot be None")) - if ref is None: - raise ValueError(_("Datastore reference cannot be None")) - if freespace is not None and capacity is None: - raise ValueError(_("Invalid capacity")) - if capacity is not None and freespace is not None: - if capacity < freespace: - raise ValueError(_("Capacity is smaller than free space")) - - self.ref = ref - self.name = name - self.capacity = capacity - self.freespace = freespace - self.type = type - self.datacenter = datacenter - - def build_path(self, *paths): - """Constructs and returns a DatastorePath. - - :param paths: list of path components, for constructing a path relative - to the root directory of the datastore - :return: a DatastorePath object - """ - return DatastorePath(self.name, *paths) - - def build_url(self, scheme, server, rel_path, datacenter_name=None): - """Constructs and returns a DatastoreURL. - - :param scheme: scheme of the URL (http, https). - :param server: hostname or ip - :param rel_path: relative path of the file on the datastore - :param datacenter_name: (optional) datacenter name - :return: a DatastoreURL object - """ - if self.datacenter is None and datacenter_name is None: - raise ValueError(_("datacenter must be set to build url")) - if datacenter_name is None: - datacenter_name = self.datacenter.name - return DatastoreURL(scheme, server, rel_path, datacenter_name, - self.name) - - def __str__(self): - return '[%s]' % self._name - - def get_summary(self, session): - """Get datastore summary. - - :param datastore: Reference to the datastore - :return: 'summary' property of the datastore - """ - return session.invoke_api(vim_util, 'get_object_property', - session.vim, self.ref, 'summary') - - def get_connected_hosts(self, session): - """Get a list of usable (accessible, mounted, read-writable) hosts where - the datastore is mounted. - - :param: session: session - :return: list of HostSystem managed object references - """ - hosts = [] - summary = self.get_summary(session) - if not summary.accessible: - return hosts - host_mounts = session.invoke_api(vim_util, 'get_object_property', - session.vim, self.ref, 'host') - if not hasattr(host_mounts, 'DatastoreHostMount'): - return hosts - for host_mount in host_mounts.DatastoreHostMount: - if self.is_datastore_mount_usable(host_mount.mountInfo): - hosts.append(host_mount.key) - return hosts - - @staticmethod - def is_datastore_mount_usable(mount_info): - """Check if a datastore is usable as per the given mount info. - - The datastore is considered to be usable for a host only if it is - writable, mounted and accessible. - - :param mount_info: HostMountInfo data object - :return: True if datastore is usable - """ - writable = mount_info.accessMode == 'readWrite' - mounted = getattr(mount_info, 'mounted', True) - accessible = getattr(mount_info, 'accessible', False) - - return writable and mounted and accessible - - @staticmethod - def choose_host(hosts): - i = random.randrange(0, len(hosts)) - return hosts[i] - - -class DatastorePath(object): - - """Class for representing a directory or file path in a vSphere datatore. - - This provides various helper methods to access components and useful - variants of the datastore path. - - Example usage: - - DatastorePath("datastore1", "_base/foo", "foo.vmdk") creates an - object that describes the "[datastore1] _base/foo/foo.vmdk" datastore - file path to a virtual disk. - - Note: - - Datastore path representations always uses forward slash as separator - (hence the use of the posixpath module). - - Datastore names are enclosed in square brackets. - - Path part of datastore path is relative to the root directory - of the datastore, and is always separated from the [ds_name] part with - a single space. - """ - - def __init__(self, datastore_name, *paths): - if datastore_name is None or datastore_name == '': - raise ValueError(_("Datastore name cannot be empty")) - self._datastore_name = datastore_name - self._rel_path = '' - if paths: - if None in paths: - raise ValueError(_("Path component cannot be None")) - self._rel_path = posixpath.join(*paths) - - def __str__(self): - """Full datastore path to the file or directory.""" - if self._rel_path != '': - return "[%s] %s" % (self._datastore_name, self.rel_path) - return "[%s]" % self._datastore_name - - @property - def datastore(self): - return self._datastore_name - - @property - def parent(self): - return DatastorePath(self.datastore, posixpath.dirname(self._rel_path)) - - @property - def basename(self): - return posixpath.basename(self._rel_path) - - @property - def dirname(self): - return posixpath.dirname(self._rel_path) - - @property - def rel_path(self): - return self._rel_path - - def join(self, *paths): - """Join one or more path components intelligently into a datastore path. - - If any component is an absolute path, all previous components are - thrown away, and joining continues. The return value is the - concatenation of the paths with exactly one slash ('/') inserted - between components, unless p is empty. - - :return: A datastore path - """ - if paths: - if None in paths: - raise ValueError(_("Path component cannot be None")) - return DatastorePath(self.datastore, self._rel_path, *paths) - return self - - def __eq__(self, other): - return (isinstance(other, DatastorePath) and - self._datastore_name == other._datastore_name and - self._rel_path == other._rel_path) - - @classmethod - def parse(cls, datastore_path): - """Constructs a DatastorePath object given a datastore path string.""" - if not datastore_path: - raise ValueError(_("Datastore path cannot be empty")) - - spl = datastore_path.split('[', 1)[1].split(']', 1) - path = "" - if len(spl) == 1: - datastore_name = spl[0] - else: - datastore_name, path = spl - return cls(datastore_name, path.strip()) - - -class DatastoreURL(object): - - """Class for representing a URL to HTTP access a file in a datastore. - - This provides various helper methods to access components and useful - variants of the datastore URL. - """ - - def __init__(self, scheme, server, path, datacenter_path, datastore_name): - self._scheme = scheme - self._server = server - self._path = path - self._datacenter_path = datacenter_path - self._datastore_name = datastore_name - params = {'dcPath': self._datacenter_path, - 'dsName': self._datastore_name} - self._query = urlparse.urlencode(params) - - @classmethod - def urlparse(cls, url): - scheme, server, path, params, query, fragment = urlparse.urlparse(url) - if not query: - path = path.split('?') - query = path[1] - path = path[0] - params = urlparse.parse_qs(query) - dc_path = params.get('dcPath') - if dc_path is not None and len(dc_path) > 0: - datacenter_path = dc_path[0] - ds_name = params.get('dsName') - if ds_name is not None and len(ds_name) > 0: - datastore_name = ds_name[0] - path = path[len('/folder'):] - return cls(scheme, server, path, datacenter_path, datastore_name) - - @property - def path(self): - return self._path.strip('/') - - @property - def datacenter_path(self): - return self._datacenter_path - - @property - def datastore_name(self): - return self._datastore_name - - def __str__(self): - return '%s://%s/folder/%s?%s' % (self._scheme, self._server, - self.path, self._query) - - def connect(self, method, content_length, cookie): - try: - if self._scheme == 'http': - conn = httplib.HTTPConnection(self._server) - elif self._scheme == 'https': - conn = httplib.HTTPSConnection(self._server) - else: - excep_msg = _("Invalid scheme: %s.") % self._scheme - LOG.error(excep_msg) - raise ValueError(excep_msg) - conn.putrequest(method, '/folder/%s?%s' % (self.path, self._query)) - conn.putheader('User-Agent', constants.USER_AGENT) - conn.putheader('Content-Length', content_length) - conn.putheader('Cookie', cookie) - conn.endheaders() - LOG.debug("Created HTTP connection to transfer the file with " - "URL = %s.", str(self)) - return conn - except (httplib.InvalidURL, httplib.CannotSendRequest, - httplib.CannotSendHeader) as excep: - excep_msg = _("Error occurred while creating HTTP connection " - "to write to file with URL = %s.") % str(self) - LOG.exception(excep_msg) - raise exceptions.VimConnectionException(excep_msg, excep) - - def get_transfer_ticket(self, session, method): - client_factory = session.vim.client.factory - spec = vim_util.get_http_service_request_spec(client_factory, method, - str(self)) - ticket = session.invoke_api( - session.vim, - 'AcquireGenericServiceTicket', - session.vim.service_content.sessionManager, - spec=spec) - return '%s="%s"' % (constants.CGI_COOKIE_KEY, ticket.id) +from oslo_vmware.objects.datastore import * # noqa diff --git a/oslo/vmware/pbm.py b/oslo/vmware/pbm.py index 5e695446..a1ba05c8 100644 --- a/oslo/vmware/pbm.py +++ b/oslo/vmware/pbm.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,188 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -VMware PBM service client and PBM related utility methods - -PBM is used for policy based placement in VMware datastores. -Refer http://goo.gl/GR2o6U for more details. -""" - -import logging -import os - -import six.moves.urllib.parse as urlparse -import six.moves.urllib.request as urllib -import suds.sax.element as element - -from oslo.vmware._i18n import _LW -from oslo.vmware import service -from oslo.vmware import vim_util - - -SERVICE_TYPE = 'PbmServiceInstance' - -LOG = logging.getLogger(__name__) - - -class Pbm(service.Service): - """Service class that provides access to the Storage Policy API.""" - - def __init__(self, protocol='https', host='localhost', port=443, - wsdl_url=None, cacert=None, insecure=True): - """Constructs a PBM service client object. - - :param protocol: http or https - :param host: server IP address or host name - :param port: port for connection - :param wsdl_url: PBM WSDL url - :param cacert: Specify a CA bundle file to use in verifying a - TLS (https) server certificate. - :param insecure: Verify HTTPS connections using system certificates, - used only if cacert is not specified - """ - base_url = service.Service.build_base_url(protocol, host, port) - soap_url = base_url + '/pbm' - super(Pbm, self).__init__(wsdl_url, soap_url, cacert, insecure) - - def set_soap_cookie(self, cookie): - """Set the specified vCenter session cookie in the SOAP header - - :param cookie: cookie to set - """ - elem = element.Element('vcSessionCookie').setText(cookie) - self.client.set_options(soapheaders=elem) - - def retrieve_service_content(self): - ref = vim_util.get_moref(service.SERVICE_INSTANCE, SERVICE_TYPE) - return self.PbmRetrieveServiceContent(ref) - - def __repr__(self): - return "PBM Object" - - def __str__(self): - return "PBM Object" - - -def get_all_profiles(session): - """Get all the profiles defined in VC server. - - :returns: PbmProfile data objects - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - LOG.debug("Fetching all the profiles defined in VC server.") - - pbm = session.pbm - profile_manager = pbm.service_content.profileManager - res_type = pbm.client.factory.create('ns0:PbmProfileResourceType') - res_type.resourceType = 'STORAGE' - profiles = [] - profile_ids = session.invoke_api(pbm, - 'PbmQueryProfile', - profile_manager, - resourceType=res_type) - LOG.debug("Fetched profile IDs: %s.", profile_ids) - if profile_ids: - profiles = session.invoke_api(pbm, - 'PbmRetrieveContent', - profile_manager, - profileIds=profile_ids) - return profiles - - -def get_profile_id_by_name(session, profile_name): - """Get the profile UUID corresponding to the given profile name. - - :param profile_name: profile name whose UUID needs to be retrieved - :returns: profile UUID string or None if profile not found - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - LOG.debug("Retrieving profile ID for profile: %s.", profile_name) - for profile in get_all_profiles(session): - if profile.name == profile_name: - profile_id = profile.profileId - LOG.debug("Retrieved profile ID: %(id)s for profile: %(name)s.", - {'id': profile_id, - 'name': profile_name}) - return profile_id - return None - - -def filter_hubs_by_profile(session, hubs, profile_id): - """Filter and return hubs that match the given profile. - - :param hubs: PbmPlacementHub morefs - :param profile_id: profile ID - :returns: subset of hubs that match the given profile - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - LOG.debug("Filtering hubs: %(hubs)s that match profile: %(profile)s.", - {'hubs': hubs, - 'profile': profile_id}) - - pbm = session.pbm - placement_solver = pbm.service_content.placementSolver - filtered_hubs = session.invoke_api(pbm, - 'PbmQueryMatchingHub', - placement_solver, - hubsToSearch=hubs, - profile=profile_id) - LOG.debug("Filtered hubs: %s", filtered_hubs) - return filtered_hubs - - -def convert_datastores_to_hubs(pbm_client_factory, datastores): - """Convert given datastore morefs to PbmPlacementHub morefs. - - :param pbm_client_factory: Factory to create PBM API input specs - :param datastores: list of datastore morefs - :returns: list of PbmPlacementHub morefs - """ - hubs = [] - for ds in datastores: - hub = pbm_client_factory.create('ns0:PbmPlacementHub') - hub.hubId = ds.value - hub.hubType = 'Datastore' - hubs.append(hub) - return hubs - - -def filter_datastores_by_hubs(hubs, datastores): - """Get filtered subset of datastores corresponding to the given hub list. - - :param hubs: list of PbmPlacementHub morefs - :param datastores: all candidate datastores - :returns: subset of datastores corresponding to the given hub list - """ - filtered_dss = [] - hub_ids = [hub.hubId for hub in hubs] - for ds in datastores: - if ds.value in hub_ids: - filtered_dss.append(ds) - return filtered_dss - - -def get_pbm_wsdl_location(vc_version): - """Return PBM WSDL file location corresponding to VC version. - - :param vc_version: a dot-separated version string. For example, "1.2". - :return: the pbm wsdl file location. - """ - if not vc_version: - return - ver = vc_version.split('.') - major_minor = ver[0] - if len(ver) >= 2: - major_minor = '%s.%s' % (major_minor, ver[1]) - curr_dir = os.path.abspath(os.path.dirname(__file__)) - pbm_service_wsdl = os.path.join(curr_dir, 'wsdl', major_minor, - 'pbmService.wsdl') - if not os.path.exists(pbm_service_wsdl): - LOG.warn(_LW("PBM WSDL file %s not found."), pbm_service_wsdl) - return - pbm_wsdl = urlparse.urljoin('file:', urllib.pathname2url(pbm_service_wsdl)) - LOG.debug("Using PBM WSDL location: %s.", pbm_wsdl) - return pbm_wsdl +from oslo_vmware.pbm import * # noqa diff --git a/oslo/vmware/rw_handles.py b/oslo/vmware/rw_handles.py index 51826858..2b79b52f 100644 --- a/oslo/vmware/rw_handles.py +++ b/oslo/vmware/rw_handles.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,620 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Classes defining read and write handles for image transfer. - -This module defines various classes for reading and writing files including -VMDK files in VMware servers. It also contains a class to read images from -glance server. -""" - -import logging -import ssl - -import requests -import six -import six.moves.urllib.parse as urlparse -from urllib3 import connection as httplib - -from oslo.utils import excutils -from oslo.utils import netutils -from oslo.vmware._i18n import _, _LE, _LW -from oslo.vmware import exceptions -from oslo.vmware import vim_util - - -LOG = logging.getLogger(__name__) - -MIN_PROGRESS_DIFF_TO_LOG = 25 -READ_CHUNKSIZE = 65536 -USER_AGENT = 'OpenStack-ESX-Adapter' - - -class FileHandle(object): - """Base class for VMware server file (including VMDK) access over HTTP. - - This class wraps a backing file handle and provides utility methods - for various sub-classes. - """ - - def __init__(self, file_handle): - """Initializes the file handle. - - :param file_handle: backing file handle - """ - self._eof = False - self._file_handle = file_handle - self._last_logged_progress = 0 - - def _create_read_connection(self, url, cookies=None, cacerts=False): - LOG.debug("Opening URL: %s for reading.", url) - try: - headers = {'User-Agent': USER_AGENT} - if cookies: - headers.update({'Cookie': - self._build_vim_cookie_header(cookies)}) - response = requests.get(url, headers=headers, stream=True, - verify=cacerts) - return response.raw - except Exception as excep: - # TODO(vbala) We need to catch and raise specific exceptions - # related to connection problems, invalid request and invalid - # arguments. - excep_msg = _("Error occurred while opening URL: %s for " - "reading.") % url - LOG.exception(excep_msg) - raise exceptions.VimException(excep_msg, excep) - - def _create_write_connection(self, url, - file_size=None, - cookies=None, - overwrite=None, - content_type=None, - cacerts=False): - """Create HTTP connection to write to VMDK file.""" - LOG.debug("Creating HTTP connection to write to file with " - "size = %(file_size)d and URL = %(url)s.", - {'file_size': file_size, - 'url': url}) - _urlparse = urlparse.urlparse(url) - scheme, netloc, path, params, query, fragment = _urlparse - - try: - if scheme == 'http': - conn = httplib.HTTPConnection(netloc) - elif scheme == 'https': - conn = httplib.HTTPSConnection(netloc) - cert_reqs = None - - # cacerts can be either True or False or contain - # actual certificates. If it is a boolean, then - # we need to set cert_reqs and clear the cacerts - if isinstance(cacerts, bool): - if cacerts: - cert_reqs = ssl.CERT_REQUIRED - else: - cert_reqs = ssl.CERT_NONE - cacerts = None - - conn.set_cert(ca_certs=cacerts, cert_reqs=cert_reqs) - else: - excep_msg = _("Invalid scheme: %s.") % scheme - LOG.error(excep_msg) - raise ValueError(excep_msg) - - if query: - path = path + '?' + query - - headers = {'User-Agent': USER_AGENT} - if file_size: - headers.update({'Content-Length': str(file_size)}) - if overwrite: - headers.update({'Overwrite': overwrite}) - if cookies: - headers.update({'Cookie': - self._build_vim_cookie_header(cookies)}) - if content_type: - headers.update({'Content-Type': content_type}) - - conn.putrequest('PUT', path) - for key, value in six.iteritems(headers): - conn.putheader(key, value) - conn.endheaders() - return conn - except requests.RequestException as excep: - excep_msg = _("Error occurred while creating HTTP connection " - "to write to VMDK file with URL = %s.") % url - LOG.exception(excep_msg) - raise exceptions.VimConnectionException(excep_msg, excep) - - def close(self): - """Close the file handle.""" - try: - self._file_handle.close() - except Exception: - LOG.warn(_LW("Error occurred while closing the file handle"), - exc_info=True) - - def _build_vim_cookie_header(self, vim_cookies): - """Build ESX host session cookie header.""" - cookie_header = "" - for vim_cookie in vim_cookies: - cookie_header = vim_cookie.name + '=' + vim_cookie.value - break - return cookie_header - - def write(self, data): - """Write data to the file. - - :param data: data to be written - :raises: NotImplementedError - """ - raise NotImplementedError() - - def read(self, chunk_size): - """Read a chunk of data. - - :param chunk_size: read chunk size - :raises: NotImplementedError - """ - raise NotImplementedError() - - def get_size(self): - """Get size of the file to be read. - - :raises: NotImplementedError - """ - raise NotImplementedError() - - def _get_soap_url(self, scheme, host, port): - """Returns the IPv4/v6 compatible SOAP URL for the given host.""" - if netutils.is_valid_ipv6(host): - return '%s://[%s]:%d' % (scheme, host, port) - return '%s://%s:%d' % (scheme, host, port) - - def _fix_esx_url(self, url, host, port): - """Fix netloc in the case of an ESX host. - - In the case of an ESX host, the netloc is set to '*' in the URL - returned in HttpNfcLeaseInfo. It should be replaced with host name - or IP address. - """ - urlp = urlparse.urlparse(url) - if urlp.netloc == '*': - scheme, netloc, path, params, query, fragment = urlp - if netutils.is_valid_ipv6(host): - netloc = '[%s]:%d' % (host, port) - else: - netloc = "%s:%d" % (host, port) - url = urlparse.urlunparse((scheme, - netloc, - path, - params, - query, - fragment)) - return url - - def _find_vmdk_url(self, lease_info, host, port): - """Find the URL corresponding to a VMDK file in lease info.""" - url = None - for deviceUrl in lease_info.deviceUrl: - if deviceUrl.disk: - url = self._fix_esx_url(deviceUrl.url, host, port) - break - if not url: - excep_msg = _("Could not retrieve VMDK URL from lease info.") - LOG.error(excep_msg) - raise exceptions.VimException(excep_msg) - LOG.debug("Found VMDK URL: %s from lease info.", url) - return url - - def _log_progress(self, progress): - """Log data transfer progress.""" - if (progress == 100 or (progress - self._last_logged_progress >= - MIN_PROGRESS_DIFF_TO_LOG)): - LOG.debug("Data transfer progress is %d%%.", progress) - self._last_logged_progress = progress - - -class FileWriteHandle(FileHandle): - """Write handle for a file in VMware server.""" - - def __init__(self, host, port, data_center_name, datastore_name, cookies, - file_path, file_size, scheme='https', cacerts=False): - """Initializes the write handle with given parameters. - - :param host: ESX/VC server IP address or host name - :param port: port for connection - :param data_center_name: name of the data center in the case of a VC - server - :param datastore_name: name of the datastore where the file is stored - :param cookies: cookies to build the vim cookie header - :param file_path: datastore path where the file is written - :param file_size: size of the file in bytes - :param scheme: protocol-- http or https - :raises: VimConnectionException, ValueError - """ - soap_url = self._get_soap_url(scheme, host, port) - param_list = {'dcPath': data_center_name, 'dsName': datastore_name} - self._url = '%s/folder/%s' % (soap_url, file_path) - self._url = self._url + '?' + urlparse.urlencode(param_list) - - self._conn = self._create_write_connection(self._url, - file_size, - cookies=cookies, - cacerts=cacerts) - FileHandle.__init__(self, self._conn) - - def write(self, data): - """Write data to the file. - - :param data: data to be written - :raises: VimConnectionException, VimException - """ - try: - self._file_handle.send(data) - except requests.RequestException as excep: - excep_msg = _("Connection error occurred while writing data to" - " %s.") % self._url - LOG.exception(excep_msg) - raise exceptions.VimConnectionException(excep_msg, excep) - except Exception as excep: - # TODO(vbala) We need to catch and raise specific exceptions - # related to connection problems, invalid request and invalid - # arguments. - excep_msg = _("Error occurred while writing data to" - " %s.") % self._url - LOG.exception(excep_msg) - raise exceptions.VimException(excep_msg, excep) - - def close(self): - """Get the response and close the connection.""" - LOG.debug("Closing write handle for %s.", self._url) - try: - self._conn.getresponse() - except Exception: - LOG.warn(_LW("Error occurred while reading the HTTP response."), - exc_info=True) - super(FileWriteHandle, self).close() - - def __str__(self): - return "File write handle for %s" % self._url - - -class VmdkWriteHandle(FileHandle): - """VMDK write handle based on HttpNfcLease. - - This class creates a vApp in the specified resource pool and uploads the - virtual disk contents. - """ - - def __init__(self, session, host, port, rp_ref, vm_folder_ref, import_spec, - vmdk_size): - """Initializes the VMDK write handle with input parameters. - - :param session: valid API session to ESX/VC server - :param host: ESX/VC server IP address or host name - :param port: port for connection - :param rp_ref: resource pool into which the backing VM is imported - :param vm_folder_ref: VM folder in ESX/VC inventory to use as parent - of backing VM - :param import_spec: import specification of the backing VM - :param vmdk_size: size of the backing VM's VMDK file - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException, - ValueError - """ - self._session = session - self._vmdk_size = vmdk_size - self._bytes_written = 0 - - # Get lease and its info for vApp import - self._lease = self._create_and_wait_for_lease(session, - rp_ref, - import_spec, - vm_folder_ref) - LOG.debug("Invoking VIM API for reading info of lease: %s.", - self._lease) - lease_info = session.invoke_api(vim_util, - 'get_object_property', - session.vim, - self._lease, - 'info') - - # Find VMDK URL where data is to be written - self._url = self._find_vmdk_url(lease_info, host, port) - self._vm_ref = lease_info.entity - - cookies = session.vim.client.options.transport.cookiejar - # Create HTTP connection to write to VMDK URL - octet_stream = 'binary/octet-stream' - self._conn = self._create_write_connection(self._url, - vmdk_size, - cookies=cookies, - overwrite='t', - content_type=octet_stream, - cacerts=session._cacert) - FileHandle.__init__(self, self._conn) - - def get_imported_vm(self): - """"Get managed object reference of the VM created for import.""" - return self._vm_ref - - def _create_and_wait_for_lease(self, session, rp_ref, import_spec, - vm_folder_ref): - """Create and wait for HttpNfcLease lease for vApp import.""" - LOG.debug("Creating HttpNfcLease lease for vApp import into resource" - " pool: %s.", - rp_ref) - lease = session.invoke_api(session.vim, - 'ImportVApp', - rp_ref, - spec=import_spec, - folder=vm_folder_ref) - LOG.debug("Lease: %(lease)s obtained for vApp import into resource" - " pool %(rp_ref)s.", - {'lease': lease, - 'rp_ref': rp_ref}) - session.wait_for_lease_ready(lease) - return lease - - def write(self, data): - """Write data to the file. - - :param data: data to be written - :raises: VimConnectionException, VimException - """ - try: - self._file_handle.send(data) - self._bytes_written += len(data) - except requests.RequestException as excep: - excep_msg = _("Connection error occurred while writing data to" - " %s.") % self._url - LOG.exception(excep_msg) - raise exceptions.VimConnectionException(excep_msg, excep) - except Exception as excep: - # TODO(vbala) We need to catch and raise specific exceptions - # related to connection problems, invalid request and invalid - # arguments. - excep_msg = _("Error occurred while writing data to" - " %s.") % self._url - LOG.exception(excep_msg) - raise exceptions.VimException(excep_msg, excep) - - # TODO(vbala) Move this method to FileHandle. - def update_progress(self): - """Updates progress to lease. - - This call back to the lease is essential to keep the lease alive - across long running write operations. - - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - progress = int(float(self._bytes_written) / self._vmdk_size * 100) - self._log_progress(progress) - - try: - self._session.invoke_api(self._session.vim, - 'HttpNfcLeaseProgress', - self._lease, - percent=progress) - except exceptions.VimException: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE("Error occurred while updating the " - "write progress of VMDK file with " - "URL = %s."), - self._url) - - def close(self): - """Releases the lease and close the connection. - - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - LOG.debug("Getting lease state for %s.", self._url) - try: - state = self._session.invoke_api(vim_util, - 'get_object_property', - self._session.vim, - self._lease, - 'state') - LOG.debug("Lease for %(url)s is in state: %(state)s.", - {'url': self._url, - 'state': state}) - if state == 'ready': - LOG.debug("Releasing lease for %s.", self._url) - self._session.invoke_api(self._session.vim, - 'HttpNfcLeaseComplete', - self._lease) - else: - LOG.debug("Lease for %(url)s is in state: %(state)s; no " - "need to release.", - {'url': self._url, - 'state': state}) - except exceptions.VimException: - LOG.warn(_LW("Error occurred while releasing the lease for %s."), - self._url, - exc_info=True) - super(VmdkWriteHandle, self).close() - LOG.debug("Closed VMDK write handle for %s.", self._url) - - def __str__(self): - return "VMDK write handle for %s" % self._url - - -class VmdkReadHandle(FileHandle): - """VMDK read handle based on HttpNfcLease.""" - - def __init__(self, session, host, port, vm_ref, vmdk_path, - vmdk_size): - """Initializes the VMDK read handle with the given parameters. - - During the read (export) operation, the VMDK file is converted to a - stream-optimized sparse disk format. Therefore, the size of the VMDK - file read may be smaller than the actual VMDK size. - - :param session: valid api session to ESX/VC server - :param host: ESX/VC server IP address or host name - :param port: port for connection - :param vm_ref: managed object reference of the backing VM whose VMDK - is to be exported - :param vmdk_path: path of the VMDK file to be exported - :param vmdk_size: actual size of the VMDK file - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - self._session = session - self._vmdk_size = vmdk_size - self._bytes_read = 0 - - # Obtain lease for VM export - self._lease = self._create_and_wait_for_lease(session, vm_ref) - LOG.debug("Invoking VIM API for reading info of lease: %s.", - self._lease) - lease_info = session.invoke_api(vim_util, - 'get_object_property', - session.vim, - self._lease, - 'info') - - # find URL of the VMDK file to be read and open connection - self._url = self._find_vmdk_url(lease_info, host, port) - cookies = session.vim.client.options.transport.cookiejar - cacerts = session.vim.client.options.transport.verify - self._conn = self._create_read_connection(self._url, - cookies=cookies, - cacerts=cacerts) - FileHandle.__init__(self, self._conn) - - def _create_and_wait_for_lease(self, session, vm_ref): - """Create and wait for HttpNfcLease lease for VM export.""" - LOG.debug("Creating HttpNfcLease lease for exporting VM: %s.", - vm_ref) - lease = session.invoke_api(session.vim, 'ExportVm', vm_ref) - LOG.debug("Lease: %(lease)s obtained for exporting VM: %(vm_ref)s.", - {'lease': lease, - 'vm_ref': vm_ref}) - session.wait_for_lease_ready(lease) - return lease - - def read(self, chunk_size): - """Read a chunk of data from the VMDK file. - - :param chunk_size: size of read chunk - :returns: the data - :raises: VimException - """ - try: - data = self._file_handle.read(READ_CHUNKSIZE) - self._bytes_read += len(data) - return data - except Exception as excep: - # TODO(vbala) We need to catch and raise specific exceptions - # related to connection problems, invalid request and invalid - # arguments. - excep_msg = _("Error occurred while reading data from" - " %s.") % self._url - LOG.exception(excep_msg) - raise exceptions.VimException(excep_msg, excep) - - def update_progress(self): - """Updates progress to lease. - - This call back to the lease is essential to keep the lease alive - across long running read operations. - - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - progress = int(float(self._bytes_read) / self._vmdk_size * 100) - self._log_progress(progress) - - try: - self._session.invoke_api(self._session.vim, - 'HttpNfcLeaseProgress', - self._lease, - percent=progress) - except exceptions.VimException: - with excutils.save_and_reraise_exception(): - LOG.exception(_LE("Error occurred while updating the " - "read progress of VMDK file with URL = %s."), - self._url) - - def close(self): - """Releases the lease and close the connection. - - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - LOG.debug("Getting lease state for %s.", self._url) - try: - state = self._session.invoke_api(vim_util, - 'get_object_property', - self._session.vim, - self._lease, - 'state') - LOG.debug("Lease for %(url)s is in state: %(state)s.", - {'url': self._url, - 'state': state}) - if state == 'ready': - LOG.debug("Releasing lease for %s.", self._url) - self._session.invoke_api(self._session.vim, - 'HttpNfcLeaseComplete', - self._lease) - else: - LOG.debug("Lease for %(url)s is in state: %(state)s; no " - "need to release.", - {'url': self._url, - 'state': state}) - except exceptions.VimException: - LOG.warn(_LW("Error occurred while releasing the lease for %s."), - self._url, - exc_info=True) - raise - super(VmdkReadHandle, self).close() - LOG.debug("Closed VMDK read handle for %s.", self._url) - - def __str__(self): - return "VMDK read handle for %s" % self._url - - -class ImageReadHandle(object): - """Read handle for glance images.""" - - def __init__(self, glance_read_iter): - """Initializes the read handle with given parameters. - - :param glance_read_iter: iterator to read data from glance image - """ - self._glance_read_iter = glance_read_iter - self._iter = self.get_next() - - def read(self, chunk_size): - """Read an item from the image data iterator. - - The input chunk size is ignored since the client ImageBodyIterator - uses its own chunk size. - """ - try: - data = next(self._iter) - return data - except StopIteration: - LOG.debug("Completed reading data from the image iterator.") - return "" - - def get_next(self): - """Get the next item from the image iterator.""" - for data in self._glance_read_iter: - yield data - - def close(self): - """Close the read handle. - - This is a NOP. - """ - pass - - def __str__(self): - return "Image read handle" +from oslo_vmware.rw_handles import * # noqa diff --git a/oslo/vmware/service.py b/oslo/vmware/service.py index 4021e82f..ebd958ff 100644 --- a/oslo/vmware/service.py +++ b/oslo/vmware/service.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,345 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Common classes that provide access to vSphere services. -""" - -import logging -import os - -import netaddr -import requests -import six -import six.moves.http_client as httplib -import suds -from suds import cache -from suds import client -from suds import plugin -from suds import transport - -from oslo.utils import timeutils -from oslo.vmware._i18n import _ -from oslo.vmware import exceptions -from oslo.vmware import vim_util - -CACHE_TIMEOUT = 60 * 60 # One hour cache timeout -ADDRESS_IN_USE_ERROR = 'Address already in use' -CONN_ABORT_ERROR = 'Software caused connection abort' -RESP_NOT_XML_ERROR = 'Response is "text/html", not "text/xml"' - -SERVICE_INSTANCE = 'ServiceInstance' - -LOG = logging.getLogger(__name__) - - -class ServiceMessagePlugin(plugin.MessagePlugin): - """Suds plug-in handling some special cases while calling VI SDK.""" - - def add_attribute_for_value(self, node): - """Helper to handle AnyType. - - Suds does not handle AnyType properly. But VI SDK requires type - attribute to be set when AnyType is used. - - :param node: XML value node - """ - if node.name == 'value': - node.set('xsi:type', 'xsd:string') - - def marshalled(self, context): - """Modifies the envelope document before it is sent. - - This method provides the plug-in with the opportunity to prune empty - nodes and fix nodes before sending it to the server. - - :param context: send context - """ - # Suds builds the entire request object based on the WSDL schema. - # VI SDK throws server errors if optional SOAP nodes are sent - # without values; e.g., as opposed to test. - context.envelope.prune() - context.envelope.walk(self.add_attribute_for_value) - - -class Response(six.BytesIO): - """Response with an input stream as source.""" - - def __init__(self, stream, status=200, headers=None): - self.status = status - self.headers = headers or {} - self.reason = requests.status_codes._codes.get( - status, [''])[0].upper().replace('_', ' ') - six.BytesIO.__init__(self, stream) - - @property - def _original_response(self): - return self - - @property - def msg(self): - return self - - def read(self, chunk_size, **kwargs): - return six.BytesIO.read(self, chunk_size) - - def info(self): - return self - - def get_all(self, name, default): - result = self.headers.get(name) - if not result: - return default - return [result] - - def getheaders(self, name): - return self.get_all(name, []) - - def release_conn(self): - self.close() - - -class LocalFileAdapter(requests.adapters.HTTPAdapter): - """Transport adapter for local files. - - See http://stackoverflow.com/a/22989322 - """ - - def _build_response_from_file(self, request): - file_path = request.url[7:] - with open(file_path, 'r') as f: - buff = bytearray(os.path.getsize(file_path)) - f.readinto(buff) - resp = Response(buff) - return self.build_response(request, resp) - - def send(self, request, stream=False, timeout=None, - verify=True, cert=None, proxies=None): - return self._build_response_from_file(request) - - -class RequestsTransport(transport.Transport): - def __init__(self, cacert=None, insecure=True): - transport.Transport.__init__(self) - # insecure flag is used only if cacert is not - # specified. - self.verify = cacert if cacert else not insecure - self.session = requests.Session() - self.session.mount('file:///', LocalFileAdapter()) - self.cookiejar = self.session.cookies - - def open(self, request): - resp = self.session.get(request.url, verify=self.verify) - return six.StringIO(resp.content) - - def send(self, request): - resp = self.session.post(request.url, - data=request.message, - headers=request.headers, - verify=self.verify) - return transport.Reply(resp.status_code, resp.headers, resp.content) - - -class MemoryCache(cache.ObjectCache): - def __init__(self): - self._cache = {} - - def get(self, key): - """Retrieves the value for a key or None.""" - now = timeutils.utcnow_ts() - for k in list(self._cache): - (timeout, _value) = self._cache[k] - if timeout and now >= timeout: - del self._cache[k] - - return self._cache.get(key, (0, None))[1] - - def put(self, key, value, time=CACHE_TIMEOUT): - """Sets the value for a key.""" - timeout = 0 - if time != 0: - timeout = timeutils.utcnow_ts() + time - self._cache[key] = (timeout, value) - return True - - -_CACHE = MemoryCache() - - -class Service(object): - """Base class containing common functionality for invoking vSphere - services - """ - - def __init__(self, wsdl_url=None, soap_url=None, - cacert=None, insecure=True): - self.wsdl_url = wsdl_url - self.soap_url = soap_url - LOG.debug("Creating suds client with soap_url='%s' and wsdl_url='%s'", - self.soap_url, self.wsdl_url) - transport = RequestsTransport(cacert, insecure) - self.client = client.Client(self.wsdl_url, - transport=transport, - location=self.soap_url, - plugins=[ServiceMessagePlugin()], - cache=_CACHE) - self._service_content = None - - @staticmethod - def build_base_url(protocol, host, port): - proto_str = '%s://' % protocol - host_str = '[%s]' % host if netaddr.valid_ipv6(host) else host - port_str = '' if port is None else ':%d' % port - return proto_str + host_str + port_str - - @staticmethod - def _retrieve_properties_ex_fault_checker(response): - """Checks the RetrievePropertiesEx API response for errors. - - Certain faults are sent in the SOAP body as a property of missingSet. - This method raises VimFaultException when a fault is found in the - response. - - :param response: response from RetrievePropertiesEx API call - :raises: VimFaultException - """ - fault_list = [] - details = {} - if not response: - # This is the case when the session has timed out. ESX SOAP - # server sends an empty RetrievePropertiesExResponse. Normally - # missingSet in the response objects has the specifics about - # the error, but that's not the case with a timed out idle - # session. It is as bad as a terminated session for we cannot - # use the session. Therefore setting fault to NotAuthenticated - # fault. - LOG.debug("RetrievePropertiesEx API response is empty; setting " - "fault to %s.", - exceptions.NOT_AUTHENTICATED) - fault_list = [exceptions.NOT_AUTHENTICATED] - else: - for obj_cont in response.objects: - if hasattr(obj_cont, 'missingSet'): - for missing_elem in obj_cont.missingSet: - f_type = missing_elem.fault.fault - f_name = f_type.__class__.__name__ - fault_list.append(f_name) - if f_name == exceptions.NO_PERMISSION: - details['object'] = f_type.object.value - details['privilegeId'] = f_type.privilegeId - - if fault_list: - fault_string = _("Error occurred while calling " - "RetrievePropertiesEx.") - raise exceptions.VimFaultException(fault_list, - fault_string, - details=details) - - @property - def service_content(self): - if self._service_content is None: - self._service_content = self.retrieve_service_content() - return self._service_content - - def get_http_cookie(self): - """Return the vCenter session cookie.""" - cookies = self.client.options.transport.cookiejar - for cookie in cookies: - if cookie.name.lower() == 'vmware_soap_session': - return cookie.value - - def __getattr__(self, attr_name): - """Returns the method to invoke API identified by param attr_name.""" - - def request_handler(managed_object, **kwargs): - """Handler for vSphere API calls. - - Invokes the API and parses the response for fault checking and - other errors. - - :param managed_object: managed object reference argument of the - API call - :param kwargs: keyword arguments of the API call - :returns: response of the API call - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - try: - if isinstance(managed_object, str): - # For strings, use string value for value and type - # of the managed object. - managed_object = vim_util.get_moref(managed_object, - managed_object) - if managed_object is None: - return - request = getattr(self.client.service, attr_name) - response = request(managed_object, **kwargs) - if (attr_name.lower() == 'retrievepropertiesex'): - Service._retrieve_properties_ex_fault_checker(response) - return response - except exceptions.VimFaultException: - # Catch the VimFaultException that is raised by the fault - # check of the SOAP response. - raise - - except suds.WebFault as excep: - fault_string = None - if excep.fault: - fault_string = excep.fault.faultstring - - doc = excep.document - detail = None - if doc is not None: - detail = doc.childAtPath('/detail') - if not detail: - # NOTE(arnaud): this is needed with VC 5.1 - detail = doc.childAtPath('/Envelope/Body/Fault/detail') - fault_list = [] - details = {} - if detail: - for fault in detail.getChildren(): - fault_list.append(fault.get("type")) - for child in fault.getChildren(): - details[child.name] = child.getText() - raise exceptions.VimFaultException(fault_list, fault_string, - excep, details) - - except AttributeError as excep: - raise exceptions.VimAttributeException( - _("No such SOAP method %s.") % attr_name, excep) - - except (httplib.CannotSendRequest, - httplib.ResponseNotReady, - httplib.CannotSendHeader) as excep: - raise exceptions.VimSessionOverLoadException( - _("httplib error in %s.") % attr_name, excep) - - except requests.RequestException as excep: - raise exceptions.VimConnectionException( - _("requests error in %s.") % attr_name, excep) - - except Exception as excep: - # TODO(vbala) should catch specific exceptions and raise - # appropriate VimExceptions. - - # Socket errors which need special handling; some of these - # might be caused by server API call overload. - if (six.text_type(excep).find(ADDRESS_IN_USE_ERROR) != -1 or - six.text_type(excep).find(CONN_ABORT_ERROR)) != -1: - raise exceptions.VimSessionOverLoadException( - _("Socket error in %s.") % attr_name, excep) - # Type error which needs special handling; it might be caused - # by server API call overload. - elif six.text_type(excep).find(RESP_NOT_XML_ERROR) != -1: - raise exceptions.VimSessionOverLoadException( - _("Type error in %s.") % attr_name, excep) - else: - raise exceptions.VimException( - _("Exception in %s.") % attr_name, excep) - return request_handler - - def __repr__(self): - return "vSphere object" - - def __str__(self): - return "vSphere object" +from oslo_vmware.service import * # noqa diff --git a/oslo/vmware/vim.py b/oslo/vmware/vim.py index 95a25432..0b83f7f5 100644 --- a/oslo/vmware/vim.py +++ b/oslo/vmware/vim.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,38 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -from oslo.vmware import service - - -class Vim(service.Service): - """Service class that provides access to the VIM API.""" - - def __init__(self, protocol='https', host='localhost', port=None, - wsdl_url=None, cacert=None, insecure=True): - """Constructs a VIM service client object. - - :param protocol: http or https - :param host: server IP address or host name - :param port: port for connection - :param wsdl_url: VIM WSDL url - :param cacert: Specify a CA bundle file to use in verifying a - TLS (https) server certificate. - :param insecure: Verify HTTPS connections using system certificates, - used only if cacert is not specified - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - base_url = service.Service.build_base_url(protocol, host, port) - soap_url = base_url + '/sdk' - if wsdl_url is None: - wsdl_url = soap_url + '/vimService.wsdl' - super(Vim, self).__init__(wsdl_url, soap_url, cacert, insecure) - - def retrieve_service_content(self): - return self.RetrieveServiceContent(service.SERVICE_INSTANCE) - - def __repr__(self): - return "VIM Object" - - def __str__(self): - return "VIM Object" +from oslo_vmware.vim import * # noqa diff --git a/oslo/vmware/vim_util.py b/oslo/vmware/vim_util.py index fd43ab56..05985cdd 100644 --- a/oslo/vmware/vim_util.py +++ b/oslo/vmware/vim_util.py @@ -1,6 +1,3 @@ -# Copyright (c) 2014 VMware, Inc. -# All Rights Reserved. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,474 +10,4 @@ # License for the specific language governing permissions and limitations # under the License. -""" -The VMware API utility module. -""" - -from suds import sudsobject - -from oslo.utils import timeutils - - -def get_moref(value, type_): - """Get managed object reference. - - :param value: value of the managed object - :param type_: type of the managed object - :returns: managed object reference with given value and type - """ - moref = sudsobject.Property(value) - moref._type = type_ - return moref - - -def build_selection_spec(client_factory, name): - """Builds the selection spec. - - :param client_factory: factory to get API input specs - :param name: name for the selection spec - :returns: selection spec - """ - sel_spec = client_factory.create('ns0:SelectionSpec') - sel_spec.name = name - return sel_spec - - -def build_traversal_spec(client_factory, name, type_, path, skip, select_set): - """Builds the traversal spec. - - :param client_factory: factory to get API input specs - :param name: name for the traversal spec - :param type_: type of the managed object - :param path: property path of the managed object - :param skip: whether or not to filter the object identified by param path - :param select_set: set of selection specs specifying additional objects - to filter - :returns: traversal spec - """ - traversal_spec = client_factory.create('ns0:TraversalSpec') - traversal_spec.name = name - traversal_spec.type = type_ - traversal_spec.path = path - traversal_spec.skip = skip - traversal_spec.selectSet = select_set - return traversal_spec - - -def build_recursive_traversal_spec(client_factory): - """Builds recursive traversal spec to traverse managed object hierarchy. - - :param client_factory: factory to get API input specs - :returns: recursive traversal spec - """ - visit_folders_select_spec = build_selection_spec(client_factory, - 'visitFolders') - # Next hop from Datacenter - dc_to_hf = build_traversal_spec(client_factory, - 'dc_to_hf', - 'Datacenter', - 'hostFolder', - False, - [visit_folders_select_spec]) - dc_to_vmf = build_traversal_spec(client_factory, - 'dc_to_vmf', - 'Datacenter', - 'vmFolder', - False, - [visit_folders_select_spec]) - dc_to_netf = build_traversal_spec(client_factory, - 'dc_to_netf', - 'Datacenter', - 'networkFolder', - False, - [visit_folders_select_spec]) - - # Next hop from HostSystem - h_to_vm = build_traversal_spec(client_factory, - 'h_to_vm', - 'HostSystem', - 'vm', - False, - [visit_folders_select_spec]) - - # Next hop from ComputeResource - cr_to_h = build_traversal_spec(client_factory, - 'cr_to_h', - 'ComputeResource', - 'host', - False, - []) - cr_to_ds = build_traversal_spec(client_factory, - 'cr_to_ds', - 'ComputeResource', - 'datastore', - False, - []) - - rp_to_rp_select_spec = build_selection_spec(client_factory, 'rp_to_rp') - rp_to_vm_select_spec = build_selection_spec(client_factory, 'rp_to_vm') - - cr_to_rp = build_traversal_spec(client_factory, - 'cr_to_rp', - 'ComputeResource', - 'resourcePool', - False, - [rp_to_rp_select_spec, - rp_to_vm_select_spec]) - - # Next hop from ClusterComputeResource - ccr_to_h = build_traversal_spec(client_factory, - 'ccr_to_h', - 'ClusterComputeResource', - 'host', - False, - []) - ccr_to_ds = build_traversal_spec(client_factory, - 'ccr_to_ds', - 'ClusterComputeResource', - 'datastore', - False, - []) - ccr_to_rp = build_traversal_spec(client_factory, - 'ccr_to_rp', - 'ClusterComputeResource', - 'resourcePool', - False, - [rp_to_rp_select_spec, - rp_to_vm_select_spec]) - # Next hop from ResourcePool - rp_to_rp = build_traversal_spec(client_factory, - 'rp_to_rp', - 'ResourcePool', - 'resourcePool', - False, - [rp_to_rp_select_spec, - rp_to_vm_select_spec]) - rp_to_vm = build_traversal_spec(client_factory, - 'rp_to_vm', - 'ResourcePool', - 'vm', - False, - [rp_to_rp_select_spec, - rp_to_vm_select_spec]) - - # Get the assorted traversal spec which takes care of the objects to - # be searched for from the rootFolder - traversal_spec = build_traversal_spec(client_factory, - 'visitFolders', - 'Folder', - 'childEntity', - False, - [visit_folders_select_spec, - h_to_vm, - dc_to_hf, - dc_to_vmf, - dc_to_netf, - cr_to_ds, - cr_to_h, - cr_to_rp, - ccr_to_h, - ccr_to_ds, - ccr_to_rp, - rp_to_rp, - rp_to_vm]) - return traversal_spec - - -def build_property_spec(client_factory, type_='VirtualMachine', - properties_to_collect=None, all_properties=False): - """Builds the property spec. - - :param client_factory: factory to get API input specs - :param type_: type of the managed object - :param properties_to_collect: names of the managed object properties to be - collected while traversal filtering - :param all_properties: whether all properties of the managed object need - to be collected - :returns: property spec - """ - if not properties_to_collect: - properties_to_collect = ['name'] - - property_spec = client_factory.create('ns0:PropertySpec') - property_spec.all = all_properties - property_spec.pathSet = properties_to_collect - property_spec.type = type_ - return property_spec - - -def build_object_spec(client_factory, root_folder, traversal_specs): - """Builds the object spec. - - :param client_factory: factory to get API input specs - :param root_folder: root folder reference; the starting point of traversal - :param traversal_specs: filter specs required for traversal - :returns: object spec - """ - object_spec = client_factory.create('ns0:ObjectSpec') - object_spec.obj = root_folder - object_spec.skip = False - object_spec.selectSet = traversal_specs - return object_spec - - -def build_property_filter_spec(client_factory, property_specs, object_specs): - """Builds the property filter spec. - - :param client_factory: factory to get API input specs - :param property_specs: property specs to be collected for filtered objects - :param object_specs: object specs to identify objects to be filtered - :returns: property filter spec - """ - property_filter_spec = client_factory.create('ns0:PropertyFilterSpec') - property_filter_spec.propSet = property_specs - property_filter_spec.objectSet = object_specs - return property_filter_spec - - -def get_objects(vim, type_, max_objects, properties_to_collect=None, - all_properties=False): - """Get all managed object references of the given type. - - It is the caller's responsibility to continue or cancel retrieval. - - :param vim: Vim object - :param type_: type of the managed object - :param max_objects: maximum number of objects that should be returned in - a single call - :param properties_to_collect: names of the managed object properties to be - collected - :param all_properties: whether all properties of the managed object need to - be collected - :returns: all managed object references of the given type - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - if not properties_to_collect: - properties_to_collect = ['name'] - - client_factory = vim.client.factory - recur_trav_spec = build_recursive_traversal_spec(client_factory) - object_spec = build_object_spec(client_factory, - vim.service_content.rootFolder, - [recur_trav_spec]) - property_spec = build_property_spec( - client_factory, - type_=type_, - properties_to_collect=properties_to_collect, - all_properties=all_properties) - property_filter_spec = build_property_filter_spec(client_factory, - [property_spec], - [object_spec]) - options = client_factory.create('ns0:RetrieveOptions') - options.maxObjects = max_objects - return vim.RetrievePropertiesEx(vim.service_content.propertyCollector, - specSet=[property_filter_spec], - options=options) - - -def get_object_properties(vim, moref, properties_to_collect): - """Get properties of the given managed object. - - :param vim: Vim object - :param moref: managed object reference - :param properties_to_collect: names of the managed object properties to be - collected - :returns: properties of the given managed object - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - if moref is None: - return None - - client_factory = vim.client.factory - all_properties = (properties_to_collect is None or - len(properties_to_collect) == 0) - property_spec = build_property_spec( - client_factory, - type_=moref._type, - properties_to_collect=properties_to_collect, - all_properties=all_properties) - object_spec = build_object_spec(client_factory, moref, []) - property_filter_spec = build_property_filter_spec(client_factory, - [property_spec], - [object_spec]) - - options = client_factory.create('ns0:RetrieveOptions') - options.maxObjects = 1 - retrieve_result = vim.RetrievePropertiesEx( - vim.service_content.propertyCollector, - specSet=[property_filter_spec], - options=options) - cancel_retrieval(vim, retrieve_result) - return retrieve_result.objects - - -def _get_token(retrieve_result): - """Get token from result to obtain next set of results. - - :retrieve_result: Result of RetrievePropertiesEx API call - :returns: token to obtain next set of results; None if no more results. - """ - return getattr(retrieve_result, 'token', None) - - -def cancel_retrieval(vim, retrieve_result): - """Cancels the retrieve operation if necessary. - - :param vim: Vim object - :param retrieve_result: result of RetrievePropertiesEx API call - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - token = _get_token(retrieve_result) - if token: - collector = vim.service_content.propertyCollector - vim.CancelRetrievePropertiesEx(collector, token=token) - - -def continue_retrieval(vim, retrieve_result): - """Continue retrieving results, if available. - - :param vim: Vim object - :param retrieve_result: result of RetrievePropertiesEx API call - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - token = _get_token(retrieve_result) - if token: - collector = vim.service_content.propertyCollector - return vim.ContinueRetrievePropertiesEx(collector, token=token) - - -def get_object_property(vim, moref, property_name): - """Get property of the given managed object. - - :param vim: Vim object - :param moref: managed object reference - :param property_name: name of the property to be retrieved - :returns: property of the given managed object - :raises: VimException, VimFaultException, VimAttributeException, - VimSessionOverLoadException, VimConnectionException - """ - props = get_object_properties(vim, moref, [property_name]) - prop_val = None - if props: - prop = None - if hasattr(props[0], 'propSet'): - # propSet will be set only if the server provides value - # for the field - prop = props[0].propSet - if prop: - prop_val = prop[0].val - return prop_val - - -def find_extension(vim, key): - """Looks for an existing extension. - - :param vim: Vim object - :param key: the key to search for - :returns: the data object Extension or None - """ - extension_manager = vim.service_content.extensionManager - return vim.client.service.FindExtension(extension_manager, key) - - -def register_extension(vim, key, type, label='OpenStack', - summary='OpenStack services', version='1.0'): - """Create a new extention. - - :param vim: Vim object - :param key: the key for the extension - :param type: Managed entity type, as defined by the extension. This - matches the type field in the configuration about a - virtual machine or vApp - :param label: Display label - :param summary: Summary description - :param version: Extension version number as a dot-separated string - """ - extension_manager = vim.service_content.extensionManager - client_factory = vim.client.factory - os_ext = client_factory.create('ns0:Extension') - os_ext.key = key - entity_info = client_factory.create('ns0:ExtManagedEntityInfo') - entity_info.type = type - os_ext.managedEntityInfo = [entity_info] - os_ext.version = version - desc = client_factory.create('ns0:Description') - desc.label = label - desc.summary = summary - os_ext.description = desc - os_ext.lastHeartbeatTime = timeutils.strtime() - vim.client.service.RegisterExtension(extension_manager, os_ext) - - -def get_vc_version(session): - """Return the dot-separated vCenter version string. For example, "1.2". - - :param session: vCenter soap session - :return: vCenter version - """ - return session.vim.service_content.about.version - - -def get_inventory_path(vim, entity_ref, max_objects=100): - """Get the inventory path of a managed entity. - - :param vim: Vim object - :param entity_ref: managed entity reference - :param max_objects: maximum number of objects that should be returned in - a single call - :return: inventory path of the entity_ref - """ - client_factory = vim.client.factory - property_collector = vim.service_content.propertyCollector - - prop_spec = build_property_spec(client_factory, 'ManagedEntity', - ['name', 'parent']) - select_set = build_selection_spec(client_factory, 'ParentTraversalSpec') - select_set = build_traversal_spec( - client_factory, 'ParentTraversalSpec', 'ManagedEntity', 'parent', - False, [select_set]) - obj_spec = build_object_spec(client_factory, entity_ref, select_set) - prop_filter_spec = build_property_filter_spec(client_factory, - [prop_spec], [obj_spec]) - options = client_factory.create('ns0:RetrieveOptions') - options.maxObjects = max_objects - retrieve_result = vim.RetrievePropertiesEx( - property_collector, - specSet=[prop_filter_spec], - options=options) - entity_name = None - propSet = None - path = "" - while retrieve_result: - for obj in retrieve_result.objects: - if hasattr(obj, 'propSet'): - propSet = obj.propSet - if len(propSet) >= 1 and not entity_name: - entity_name = propSet[0].val - elif len(propSet) >= 1: - path = '%s/%s' % (propSet[0].val, path) - retrieve_result = continue_retrieval(vim, retrieve_result) - # NOTE(arnaud): slice to exclude the root folder from the result. - if propSet is not None and len(propSet) > 0: - path = path[len(propSet[0].val):] - if entity_name is None: - entity_name = "" - return '%s%s' % (path, entity_name) - - -def get_http_service_request_spec(client_factory, method, uri): - """Build a HTTP service request spec. - - :param client_factory: factory to get API input specs - :param method: HTTP method (GET, POST, PUT) - :param uri: target URL - """ - http_service_request_spec = client_factory.create( - 'ns0:SessionManagerHttpServiceRequestSpec') - http_service_request_spec.method = method - http_service_request_spec.url = uri - return http_service_request_spec +from oslo_vmware.vim_util import * # noqa diff --git a/oslo/vmware/common/__init__.py b/oslo_vmware/__init__.py similarity index 100% rename from oslo/vmware/common/__init__.py rename to oslo_vmware/__init__.py diff --git a/oslo/vmware/_i18n.py b/oslo_vmware/_i18n.py similarity index 100% rename from oslo/vmware/_i18n.py rename to oslo_vmware/_i18n.py diff --git a/oslo_vmware/api.py b/oslo_vmware/api.py new file mode 100644 index 00000000..61886faa --- /dev/null +++ b/oslo_vmware/api.py @@ -0,0 +1,500 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Session and API call management for VMware ESX/VC server. + +This module contains classes to invoke VIM APIs. It supports +automatic session re-establishment and retry of API invocations +in case of connection problems or server API call overload. +""" + +import logging + +import six + +from oslo.utils import excutils +from oslo_vmware._i18n import _, _LE, _LI, _LW +from oslo_vmware.common import loopingcall +from oslo_vmware import exceptions +from oslo_vmware import pbm +from oslo_vmware import vim +from oslo_vmware import vim_util + + +LOG = logging.getLogger(__name__) + + +def _trunc_id(session_id): + """Returns truncated session id which is suitable for logging.""" + if session_id is not None: + return session_id[-5:] + + +# TODO(vbala) Move this class to excutils.py. +class RetryDecorator(object): + """Decorator for retrying a function upon suggested exceptions. + + The decorated function is retried for the given number of times, and the + sleep time between the retries is incremented until max sleep time is + reached. If the max retry count is set to -1, then the decorated function + is invoked indefinitely until an exception is thrown, and the caught + exception is not in the list of suggested exceptions. + """ + + def __init__(self, max_retry_count=-1, inc_sleep_time=10, + max_sleep_time=60, exceptions=()): + """Configure the retry object using the input params. + + :param max_retry_count: maximum number of times the given function must + be retried when one of the input 'exceptions' + is caught. When set to -1, it will be retried + indefinitely until an exception is thrown + and the caught exception is not in param + exceptions. + :param inc_sleep_time: incremental time in seconds for sleep time + between retries + :param max_sleep_time: max sleep time in seconds beyond which the sleep + time will not be incremented using param + inc_sleep_time. On reaching this threshold, + max_sleep_time will be used as the sleep time. + :param exceptions: suggested exceptions for which the function must be + retried + """ + self._max_retry_count = max_retry_count + self._inc_sleep_time = inc_sleep_time + self._max_sleep_time = max_sleep_time + self._exceptions = exceptions + self._retry_count = 0 + self._sleep_time = 0 + + def __call__(self, f): + + def _func(*args, **kwargs): + func_name = f.__name__ + result = None + try: + if self._retry_count: + LOG.debug("Invoking %(func_name)s; retry count is " + "%(retry_count)d.", + {'func_name': func_name, + 'retry_count': self._retry_count}) + result = f(*args, **kwargs) + except self._exceptions: + with excutils.save_and_reraise_exception() as ctxt: + LOG.warn(_LW("Exception which is in the suggested list of " + "exceptions occurred while invoking function:" + " %s."), + func_name, + exc_info=True) + if (self._max_retry_count != -1 and + self._retry_count >= self._max_retry_count): + LOG.error(_LE("Cannot retry upon suggested exception " + "since retry count (%(retry_count)d) " + "reached max retry count " + "(%(max_retry_count)d)."), + {'retry_count': self._retry_count, + 'max_retry_count': self._max_retry_count}) + else: + ctxt.reraise = False + self._retry_count += 1 + self._sleep_time += self._inc_sleep_time + return self._sleep_time + raise loopingcall.LoopingCallDone(result) + + def func(*args, **kwargs): + loop = loopingcall.DynamicLoopingCall(_func, *args, **kwargs) + evt = loop.start(periodic_interval_max=self._max_sleep_time) + LOG.debug("Waiting for function %s to return.", f.__name__) + return evt.wait() + + return func + + +class VMwareAPISession(object): + """Setup a session with the server and handles all calls made to it. + + Example: + api_session = VMwareAPISession('10.1.2.3', 'administrator', + 'password', 10, 0.1, + create_session=False, port=443) + result = api_session.invoke_api(vim_util, 'get_objects', + api_session.vim, 'HostSystem', 100) + """ + + def __init__(self, host, server_username, server_password, + api_retry_count, task_poll_interval, scheme='https', + create_session=True, wsdl_loc=None, pbm_wsdl_loc=None, + port=443, cacert=None, insecure=True): + """Initializes the API session with given parameters. + + :param host: ESX/VC server IP address or host name + :param port: port for connection + :param server_username: username of ESX/VC server admin user + :param server_password: password for param server_username + :param api_retry_count: number of times an API must be retried upon + session/connection related errors + :param task_poll_interval: sleep time in seconds for polling an + on-going async task as part of the API call + :param scheme: protocol-- http or https + :param create_session: whether to setup a connection at the time of + instance creation + :param wsdl_loc: VIM API WSDL file location + :param pbm_wsdl_loc: PBM service WSDL file location + :param cacert: Specify a CA bundle file to use in verifying a + TLS (https) server certificate. + :param insecure: Verify HTTPS connections using system certificates, + used only if cacert is not specified + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException + """ + self._host = host + self._port = port + self._server_username = server_username + self._server_password = server_password + self._api_retry_count = api_retry_count + self._task_poll_interval = task_poll_interval + self._scheme = scheme + self._vim_wsdl_loc = wsdl_loc + self._pbm_wsdl_loc = pbm_wsdl_loc + self._session_id = None + self._session_username = None + self._vim = None + self._pbm = None + self._cacert = cacert + self._insecure = insecure + if create_session: + self._create_session() + + def pbm_wsdl_loc_set(self, pbm_wsdl_loc): + self._pbm_wsdl_loc = pbm_wsdl_loc + self._pbm = None + LOG.info(_LI('PBM WSDL updated to %s'), pbm_wsdl_loc) + + @property + def vim(self): + if not self._vim: + self._vim = vim.Vim(protocol=self._scheme, + host=self._host, + port=self._port, + wsdl_url=self._vim_wsdl_loc, + cacert=self._cacert, + insecure=self._insecure) + return self._vim + + @property + def pbm(self): + if not self._pbm and self._pbm_wsdl_loc: + self._pbm = pbm.Pbm(protocol=self._scheme, + host=self._host, + port=self._port, + wsdl_url=self._pbm_wsdl_loc, + cacert=self._cacert, + insecure=self._insecure) + if self._session_id: + # To handle the case where pbm property is accessed after + # session creation. If pbm property is accessed before session + # creation, we set the cookie in _create_session. + self._pbm.set_soap_cookie(self._vim.get_http_cookie()) + return self._pbm + + @RetryDecorator(exceptions=(exceptions.VimConnectionException,)) + def _create_session(self): + """Establish session with the server.""" + session_manager = self.vim.service_content.sessionManager + # Login and create new session with the server for making API calls. + LOG.debug("Logging in with username = %s.", self._server_username) + session = self.vim.Login(session_manager, + userName=self._server_username, + password=self._server_password) + prev_session_id, self._session_id = self._session_id, session.key + # We need to save the username in the session since we may need it + # later to check active session. The SessionIsActive method requires + # the username parameter to be exactly same as that in the session + # object. We can't use the username used for login since the Login + # method ignores the case. + self._session_username = session.userName + LOG.info(_LI("Successfully established new session; session ID is " + "%s."), + _trunc_id(self._session_id)) + + # Terminate the previous session (if exists) for preserving sessions + # as there is a limit on the number of sessions we can have. + if prev_session_id: + try: + LOG.info(_LI("Terminating the previous session with ID = %s"), + _trunc_id(prev_session_id)) + self.vim.TerminateSession(session_manager, + sessionId=[prev_session_id]) + except Exception: + # This exception is something we can live with. It is + # just an extra caution on our side. The session might + # have been cleared already. We could have made a call to + # SessionIsActive, but that is an overhead because we + # anyway would have to call TerminateSession. + LOG.warn(_LW("Error occurred while terminating the previous " + "session with ID = %s."), + _trunc_id(prev_session_id), + exc_info=True) + + # Set PBM client cookie. + if self._pbm is not None: + self._pbm.set_soap_cookie(self._vim.get_http_cookie()) + + def logout(self): + """Log out and terminate the current session.""" + if self._session_id: + LOG.info(_LI("Logging out and terminating the current session " + "with ID = %s."), + _trunc_id(self._session_id)) + try: + self.vim.Logout(self.vim.service_content.sessionManager) + self._session_id = None + except Exception: + LOG.exception(_LE("Error occurred while logging out and " + "terminating the current session with " + "ID = %s."), + _trunc_id(self._session_id)) + else: + LOG.debug("No session exists to log out.") + + def invoke_api(self, module, method, *args, **kwargs): + """Wrapper method for invoking APIs. + + The API call is retried in the event of exceptions due to session + overload or connection problems. + + :param module: module corresponding to the VIM API call + :param method: method in the module which corresponds to the + VIM API call + :param args: arguments to the method + :param kwargs: keyword arguments to the method + :returns: response from the API call + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + + @RetryDecorator(max_retry_count=self._api_retry_count, + exceptions=(exceptions.VimSessionOverLoadException, + exceptions.VimConnectionException)) + def _invoke_api(module, method, *args, **kwargs): + try: + api_method = getattr(module, method) + return api_method(*args, **kwargs) + except exceptions.VimFaultException as excep: + # If this is due to an inactive session, we should re-create + # the session and retry. + if exceptions.NOT_AUTHENTICATED in excep.fault_list: + # The NotAuthenticated fault is set by the fault checker + # due to an empty response. An empty response could be a + # valid response; for e.g., response for the query to + # return the VMs in an ESX server which has no VMs in it. + # Also, the server responds with an empty response in the + # case of an inactive session. Therefore, we need a way to + # differentiate between these two cases. + if self.is_current_session_active(): + LOG.debug("Returning empty response for " + "%(module)s.%(method)s invocation.", + {'module': module, + 'method': method}) + return [] + else: + # empty response is due to an inactive session + excep_msg = ( + _("Current session: %(session)s is inactive; " + "re-creating the session while invoking " + "method %(module)s.%(method)s.") % + {'session': _trunc_id(self._session_id), + 'module': module, + 'method': method}) + LOG.warn(excep_msg, exc_info=True) + self._create_session() + raise exceptions.VimConnectionException(excep_msg, + excep) + else: + # no need to retry for other VIM faults like + # InvalidArgument + # Raise specific exceptions here if possible + if excep.fault_list: + LOG.debug("Fault list: %s", excep.fault_list) + fault = excep.fault_list[0] + clazz = exceptions.get_fault_class(fault) + raise clazz(six.text_type(excep), excep.details) + raise + + except exceptions.VimConnectionException: + with excutils.save_and_reraise_exception(): + # Re-create the session during connection exception only + # if the session has expired. Otherwise, it could be + # a transient issue. + if not self.is_current_session_active(): + LOG.warn(_LW("Re-creating session due to connection " + "problems while invoking method " + "%(module)s.%(method)s."), + {'module': module, + 'method': method}, + exc_info=True) + self._create_session() + + return _invoke_api(module, method, *args, **kwargs) + + def is_current_session_active(self): + """Check if current session is active. + + :returns: True if the session is active; False otherwise + """ + LOG.debug("Checking if the current session: %s is active.", + _trunc_id(self._session_id)) + + is_active = False + try: + is_active = self.vim.SessionIsActive( + self.vim.service_content.sessionManager, + sessionID=self._session_id, + userName=self._session_username) + except exceptions.VimException: + LOG.warn(_LW("Error occurred while checking whether the " + "current session: %s is active."), + _trunc_id(self._session_id), + exc_info=True) + + return is_active + + def wait_for_task(self, task): + """Waits for the given task to complete and returns the result. + + The task is polled until it is done. The method returns the task + information upon successful completion. In case of any error, + appropriate exception is raised. + + :param task: managed object reference of the task + :returns: task info upon successful completion of the task + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + loop = loopingcall.FixedIntervalLoopingCall(self._poll_task, task) + evt = loop.start(self._task_poll_interval) + LOG.debug("Waiting for the task: %s to complete.", task) + return evt.wait() + + def _poll_task(self, task): + """Poll the given task until completion. + + If the task completes successfully, the method returns the task info + using the input event (param done). In case of any error, appropriate + exception is set in the event. + + :param task: managed object reference of the task + """ + LOG.debug("Invoking VIM API to read info of task: %s.", task) + try: + task_info = self.invoke_api(vim_util, + 'get_object_property', + self.vim, + task, + 'info') + except exceptions.VimException: + with excutils.save_and_reraise_exception(): + LOG.exception(_LE("Error occurred while reading info of " + "task: %s."), + task) + else: + if task_info.state in ['queued', 'running']: + if hasattr(task_info, 'progress'): + LOG.debug("Task: %(task)s progress is %(progress)s%%.", + {'task': task, + 'progress': task_info.progress}) + elif task_info.state == 'success': + LOG.debug("Task: %s status is success.", task) + raise loopingcall.LoopingCallDone(task_info) + else: + error_msg = six.text_type(task_info.error.localizedMessage) + error = task_info.error + name = error.fault.__class__.__name__ + task_ex = exceptions.get_fault_class(name)(error_msg) + raise task_ex + + def wait_for_lease_ready(self, lease): + """Waits for the given lease to be ready. + + This method return when the lease is ready. In case of any error, + appropriate exception is raised. + + :param lease: lease to be checked for + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + loop = loopingcall.FixedIntervalLoopingCall(self._poll_lease, lease) + evt = loop.start(self._task_poll_interval) + LOG.debug("Waiting for the lease: %s to be ready.", lease) + evt.wait() + + def _poll_lease(self, lease): + """Poll the state of the given lease. + + When the lease is ready, the event (param done) is notified. In case + of any error, appropriate exception is set in the event. + + :param lease: lease whose state is to be polled + """ + LOG.debug("Invoking VIM API to read state of lease: %s.", lease) + try: + state = self.invoke_api(vim_util, + 'get_object_property', + self.vim, + lease, + 'state') + except exceptions.VimException: + with excutils.save_and_reraise_exception(): + LOG.exception(_LE("Error occurred while checking " + "state of lease: %s."), + lease) + else: + if state == 'ready': + LOG.debug("Lease: %s is ready.", lease) + raise loopingcall.LoopingCallDone() + elif state == 'initializing': + LOG.debug("Lease: %s is initializing.", lease) + elif state == 'error': + LOG.debug("Invoking VIM API to read lease: %s error.", + lease) + error_msg = self._get_error_message(lease) + excep_msg = _("Lease: %(lease)s is in error state. Details: " + "%(error_msg)s.") % {'lease': lease, + 'error_msg': error_msg} + LOG.error(excep_msg) + raise exceptions.VimException(excep_msg) + else: + # unknown state + excep_msg = _("Unknown state: %(state)s for lease: " + "%(lease)s.") % {'state': state, + 'lease': lease} + LOG.error(excep_msg) + raise exceptions.VimException(excep_msg) + + def _get_error_message(self, lease): + """Get error message associated with the given lease.""" + try: + return self.invoke_api(vim_util, + 'get_object_property', + self.vim, + lease, + 'error') + except exceptions.VimException: + LOG.warn(_LW("Error occurred while reading error message for " + "lease: %s."), + lease, + exc_info=True) + return "Unknown" diff --git a/oslo_vmware/common/__init__.py b/oslo_vmware/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oslo/vmware/common/loopingcall.py b/oslo_vmware/common/loopingcall.py similarity index 99% rename from oslo/vmware/common/loopingcall.py rename to oslo_vmware/common/loopingcall.py index e82b5cb6..815dbcbc 100644 --- a/oslo/vmware/common/loopingcall.py +++ b/oslo_vmware/common/loopingcall.py @@ -22,7 +22,7 @@ from eventlet import event from eventlet import greenthread from oslo.utils import timeutils -from oslo.vmware._i18n import _LE, _LW +from oslo_vmware._i18n import _LE, _LW LOG = logging.getLogger(__name__) diff --git a/oslo_vmware/constants.py b/oslo_vmware/constants.py new file mode 100644 index 00000000..d166d4e3 --- /dev/null +++ b/oslo_vmware/constants.py @@ -0,0 +1,32 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Shared constants across the VMware ecosystem. +""" + +# Datacenter path for HTTP access to datastores if the target server is an ESX/ +# ESXi system: http://goo.gl/B5Htr8 for more information. +ESX_DATACENTER_PATH = 'ha-datacenter' + +# User Agent for HTTP requests between OpenStack and vCenter. +USER_AGENT = 'OpenStack-ESX-Adapter' + +# Key of the cookie header when using a SOAP session. +SOAP_COOKIE_KEY = 'vmware_soap_session' + +# Key of the cookie header when using a CGI session. +CGI_COOKIE_KEY = 'vmware_cgi_ticket' diff --git a/oslo_vmware/exceptions.py b/oslo_vmware/exceptions.py new file mode 100644 index 00000000..32c04e40 --- /dev/null +++ b/oslo_vmware/exceptions.py @@ -0,0 +1,261 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Exception definitions. +""" + +import logging + +import six + +from oslo_vmware._i18n import _, _LE + +LOG = logging.getLogger(__name__) + +ALREADY_EXISTS = 'AlreadyExists' +CANNOT_DELETE_FILE = 'CannotDeleteFile' +FILE_ALREADY_EXISTS = 'FileAlreadyExists' +FILE_FAULT = 'FileFault' +FILE_LOCKED = 'FileLocked' +FILE_NOT_FOUND = 'FileNotFound' +INVALID_POWER_STATE = 'InvalidPowerState' +INVALID_PROPERTY = 'InvalidProperty' +NO_PERMISSION = 'NoPermission' +NOT_AUTHENTICATED = 'NotAuthenticated' +TASK_IN_PROGRESS = 'TaskInProgress' +DUPLICATE_NAME = 'DuplicateName' + + +class VimException(Exception): + """The base exception class for all exceptions this library raises.""" + + if six.PY2: + __str__ = lambda self: six.text_type(self).encode('utf8') + __unicode__ = lambda self: self.description + else: + __str__ = lambda self: self.description + + def __init__(self, message, cause=None): + Exception.__init__(self) + if isinstance(message, list): + # we need this to protect against developers using + # this method like VimFaultException + raise ValueError(_("exception_summary must not be a list")) + + self.msg = message + self.cause = cause + + @property + def description(self): + # NOTE(jecarey): self.msg and self.cause may be i18n objects + # that do not support str or concatenation, but can be used + # as replacement text. + descr = six.text_type(self.msg) + if self.cause: + descr += '\nCause: ' + six.text_type(self.cause) + return descr + + +class VimSessionOverLoadException(VimException): + """Thrown when there is an API call overload at the VMware server.""" + pass + + +class VimConnectionException(VimException): + """Thrown when there is a connection problem.""" + pass + + +class VimAttributeException(VimException): + """Thrown when a particular attribute cannot be found.""" + pass + + +class VimFaultException(VimException): + """Exception thrown when there are faults during VIM API calls.""" + + def __init__(self, fault_list, message, cause=None, details=None): + super(VimFaultException, self).__init__(message, cause) + if not isinstance(fault_list, list): + raise ValueError(_("fault_list must be a list")) + if details is not None and not isinstance(details, dict): + raise ValueError(_("details must be a dict")) + self.fault_list = fault_list + self.details = details + + if six.PY2: + __unicode__ = lambda self: self.description + else: + __str__ = lambda self: self.description + + @property + def description(self): + descr = VimException.description.fget(self) + if self.fault_list: + # fault_list doesn't contain non-ASCII chars, we can use str() + descr += '\nFaults: ' + str(self.fault_list) + if self.details: + # details may contain non-ASCII values + details = '{%s}' % ', '.join(["'%s': '%s'" % (k, v) for k, v in + six.iteritems(self.details)]) + descr += '\nDetails: ' + details + return descr + + +class ImageTransferException(VimException): + """Thrown when there is an error during image transfer.""" + pass + + +class VMwareDriverException(Exception): + """Base VMware Driver Exception + + To correctly use this class, inherit from it and define + a 'msg_fmt' property. That msg_fmt will get printf'd + with the keyword arguments provided to the constructor. + + """ + msg_fmt = _("An unknown exception occurred.") + + def __init__(self, message=None, details=None, **kwargs): + self.kwargs = kwargs + self.details = details + + if not message: + try: + message = self.msg_fmt % kwargs + + except Exception: + # kwargs doesn't match a variable in the message + # log the issue and the kwargs + LOG.exception(_LE('Exception in string format operation')) + for name, value in six.iteritems(kwargs): + LOG.error(_LE("%(name)s: %(value)s"), + {'name': name, 'value': value}) + # at least get the core message out if something happened + message = self.msg_fmt + + super(VMwareDriverException, self).__init__(message) + + +class VMwareDriverConfigurationException(VMwareDriverException): + """Base class for all configuration exceptions. + """ + msg_fmt = _("VMware Driver configuration fault.") + + +class UseLinkedCloneConfigurationFault(VMwareDriverConfigurationException): + msg_fmt = _("No default value for use_linked_clone found.") + + +class MissingParameter(VMwareDriverException): + msg_fmt = _("Missing parameter : %(param)s") + + +class AlreadyExistsException(VMwareDriverException): + msg_fmt = _("Resource already exists.") + code = 409 + + +class CannotDeleteFileException(VMwareDriverException): + msg_fmt = _("Cannot delete file.") + code = 403 + + +class FileAlreadyExistsException(VMwareDriverException): + msg_fmt = _("File already exists.") + code = 409 + + +class FileFaultException(VMwareDriverException): + msg_fmt = _("File fault.") + code = 409 + + +class FileLockedException(VMwareDriverException): + msg_fmt = _("File locked.") + code = 403 + + +class FileNotFoundException(VMwareDriverException): + msg_fmt = _("File not found.") + code = 404 + + +class InvalidPowerStateException(VMwareDriverException): + msg_fmt = _("Invalid power state.") + code = 409 + + +class InvalidPropertyException(VMwareDriverException): + msg_fmt = _("Invalid property.") + code = 400 + + +class NoPermissionException(VMwareDriverException): + msg_fmt = _("No Permission.") + code = 403 + + +class NotAuthenticatedException(VMwareDriverException): + msg_fmt = _("Not Authenticated.") + code = 403 + + +class TaskInProgress(VMwareDriverException): + msg_fmt = _("Entity has another operation in process.") + + +class DuplicateName(VMwareDriverException): + msg_fmt = _("Duplicate name.") + + +# Populate the fault registry with the exceptions that have +# special treatment. +_fault_classes_registry = { + ALREADY_EXISTS: AlreadyExistsException, + CANNOT_DELETE_FILE: CannotDeleteFileException, + FILE_ALREADY_EXISTS: FileAlreadyExistsException, + FILE_FAULT: FileFaultException, + FILE_LOCKED: FileLockedException, + FILE_NOT_FOUND: FileNotFoundException, + INVALID_POWER_STATE: InvalidPowerStateException, + INVALID_PROPERTY: InvalidPropertyException, + NO_PERMISSION: NoPermissionException, + NOT_AUTHENTICATED: NotAuthenticatedException, + TASK_IN_PROGRESS: TaskInProgress, + DUPLICATE_NAME: DuplicateName, +} + + +def get_fault_class(name): + """Get a named subclass of VMwareDriverException.""" + name = str(name) + fault_class = _fault_classes_registry.get(name) + if not fault_class: + LOG.debug('Fault %s not matched.', name) + fault_class = VMwareDriverException + return fault_class + + +def register_fault_class(name, exception): + fault_class = _fault_classes_registry.get(name) + if not issubclass(exception, VMwareDriverException): + raise TypeError(_("exception should be a subclass of " + "VMwareDriverException")) + if fault_class: + LOG.debug('Overriding exception for %s', name) + _fault_classes_registry[name] = exception diff --git a/oslo_vmware/image_transfer.py b/oslo_vmware/image_transfer.py new file mode 100644 index 00000000..825cbc67 --- /dev/null +++ b/oslo_vmware/image_transfer.py @@ -0,0 +1,608 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Functions and classes for image transfer between ESX/VC & image service. +""" + +import errno +import logging + +from eventlet import event +from eventlet import greenthread +from eventlet import queue +from eventlet import timeout + +from oslo_vmware._i18n import _ +from oslo_vmware import constants +from oslo_vmware import exceptions +from oslo_vmware.objects import datastore as ds_obj +from oslo_vmware import rw_handles +from oslo_vmware import vim_util + + +LOG = logging.getLogger(__name__) + +IMAGE_SERVICE_POLL_INTERVAL = 5 +FILE_READ_WRITE_TASK_SLEEP_TIME = 0.01 +BLOCKING_QUEUE_SIZE = 10 + + +class BlockingQueue(queue.LightQueue): + """Producer-Consumer queue to share data between reader/writer threads.""" + + def __init__(self, max_size, max_transfer_size): + """Initializes the queue with the given parameters. + + :param max_size: maximum queue size; if max_size is less than zero or + None, the queue size is infinite. + :param max_transfer_size: maximum amount of data that can be + _transferred using this queue + """ + queue.LightQueue.__init__(self, max_size) + self._max_transfer_size = max_transfer_size + self._transferred = 0 + + def read(self, chunk_size): + """Read data from the queue. + + This method blocks until data is available. The input chunk size is + ignored since we have ensured that the data chunks written to the pipe + by the image reader thread is the same as the chunks asked for by the + image writer thread. + """ + if (self._max_transfer_size is 0 or + self._transferred < self._max_transfer_size): + data_item = self.get() + self._transferred += len(data_item) + return data_item + else: + LOG.debug("Completed transfer of size %s.", self._transferred) + return "" + + def write(self, data): + """Write data into the queue. + + :param data: data to be written + """ + self.put(data) + + # Below methods are provided in order to enable treating the queue + # as a file handle. + + def seek(self, offset, whence=0): + """Set the file's current position at the offset. + + This method throws IOError since seek cannot be supported for a pipe. + """ + raise IOError(errno.ESPIPE, "Illegal seek") + + def tell(self): + """Get the current file position.""" + return self._transferred + + def close(self): + pass + + def __str__(self): + return "blocking queue" + + +class ImageWriter(object): + """Class to write the image to the image service from an input file.""" + + def __init__(self, context, input_file, image_service, image_id, + image_meta=None): + """Initializes the image writer instance with given parameters. + + :param context: write context needed by the image service + :param input_file: file to read the image data from + :param image_service: handle to image service + :param image_id: ID of the image in the image service + :param image_meta: image meta-data + """ + if not image_meta: + image_meta = {} + + self._context = context + self._input_file = input_file + self._image_service = image_service + self._image_id = image_id + self._image_meta = image_meta + self._running = False + + def start(self): + """Start the image write task. + + :returns: the event indicating the status of the write task + """ + self._done = event.Event() + + def _inner(): + """Task performing the image write operation. + + This method performs image data transfer through an update call. + After the update, it waits until the image state becomes + 'active', 'killed' or unknown. If the final state is not 'active' + an instance of ImageTransferException is thrown. + + :raises: ImageTransferException + """ + LOG.debug("Calling image service update on image: %(image)s " + "with meta: %(meta)s", + {'image': self._image_id, + 'meta': self._image_meta}) + + try: + self._image_service.update(self._context, + self._image_id, + self._image_meta, + data=self._input_file) + self._running = True + while self._running: + LOG.debug("Retrieving status of image: %s.", + self._image_id) + image_meta = self._image_service.show(self._context, + self._image_id) + image_status = image_meta.get('status') + if image_status == 'active': + self.stop() + LOG.debug("Image: %s is now active.", + self._image_id) + self._done.send(True) + elif image_status == 'killed': + self.stop() + excep_msg = (_("Image: %s is in killed state.") % + self._image_id) + LOG.error(excep_msg) + excep = exceptions.ImageTransferException(excep_msg) + self._done.send_exception(excep) + elif image_status in ['saving', 'queued']: + LOG.debug("Image: %(image)s is in %(state)s state; " + "sleeping for %(sleep)d seconds.", + {'image': self._image_id, + 'state': image_status, + 'sleep': IMAGE_SERVICE_POLL_INTERVAL}) + greenthread.sleep(IMAGE_SERVICE_POLL_INTERVAL) + else: + self.stop() + excep_msg = (_("Image: %(image)s is in unknown " + "state: %(state)s.") % + {'image': self._image_id, + 'state': image_status}) + LOG.error(excep_msg) + excep = exceptions.ImageTransferException(excep_msg) + self._done.send_exception(excep) + except Exception as excep: + self.stop() + excep_msg = (_("Error occurred while writing image: %s") % + self._image_id) + LOG.exception(excep_msg) + excep = exceptions.ImageTransferException(excep_msg, excep) + self._done.send_exception(excep) + + LOG.debug("Starting image write task for image: %(image)s with" + " source: %(source)s.", + {'source': self._input_file, + 'image': self._image_id}) + greenthread.spawn(_inner) + return self._done + + def stop(self): + """Stop the image writing task.""" + LOG.debug("Stopping the writing task for image: %s.", + self._image_id) + self._running = False + + def wait(self): + """Wait for the image writer task to complete. + + This method returns True if the writer thread completes successfully. + In case of error, it raises ImageTransferException. + + :raises ImageTransferException + """ + return self._done.wait() + + def close(self): + """This is a NOP.""" + pass + + def __str__(self): + string = "Image Writer " % (self._input_file, + self._image_id) + return string + + +class FileReadWriteTask(object): + """Task which reads data from the input file and writes to the output file. + + This class defines the task which copies the given input file to the given + output file. The copy operation involves reading chunks of data from the + input file and writing the same to the output file. + """ + + def __init__(self, input_file, output_file): + """Initializes the read-write task with the given input parameters. + + :param input_file: the input file handle + :param output_file: the output file handle + """ + self._input_file = input_file + self._output_file = output_file + self._running = False + + def start(self): + """Start the file read - file write task. + + :returns: the event indicating the status of the read-write task + """ + self._done = event.Event() + + def _inner(): + """Task performing the file read-write operation.""" + self._running = True + while self._running: + try: + data = self._input_file.read(rw_handles.READ_CHUNKSIZE) + if not data: + LOG.debug("File read-write task is done.") + self.stop() + self._done.send(True) + self._output_file.write(data) + + # update lease progress if applicable + if hasattr(self._input_file, "update_progress"): + self._input_file.update_progress() + if hasattr(self._output_file, "update_progress"): + self._output_file.update_progress() + + greenthread.sleep(FILE_READ_WRITE_TASK_SLEEP_TIME) + except Exception as excep: + self.stop() + excep_msg = _("Error occurred during file read-write " + "task.") + LOG.exception(excep_msg) + excep = exceptions.ImageTransferException(excep_msg, excep) + self._done.send_exception(excep) + + LOG.debug("Starting file read-write task with source: %(source)s " + "and destination: %(dest)s.", + {'source': self._input_file, + 'dest': self._output_file}) + greenthread.spawn(_inner) + return self._done + + def stop(self): + """Stop the read-write task.""" + LOG.debug("Stopping the file read-write task.") + self._running = False + + def wait(self): + """Wait for the file read-write task to complete. + + This method returns True if the read-write thread completes + successfully. In case of error, it raises ImageTransferException. + + :raises: ImageTransferException + """ + return self._done.wait() + + def __str__(self): + string = ("File Read-Write Task " % + (self._input_file, self._output_file)) + return string + + +# Functions to perform image transfer between VMware servers and image service. + + +def _start_transfer(context, timeout_secs, read_file_handle, max_data_size, + write_file_handle=None, image_service=None, image_id=None, + image_meta=None): + """Start the image transfer. + + The image reader reads the data from the image source and writes to the + blocking queue. The image source is always a file handle (VmdkReadHandle + or ImageReadHandle); therefore, a FileReadWriteTask is created for this + transfer. The image writer reads the data from the blocking queue and + writes it to the image destination. The image destination is either a + file or VMDK in VMware datastore or an image in the image service. + + If the destination is a file or VMDK in VMware datastore, the method + creates a FileReadWriteTask which reads from the blocking queue and + writes to either FileWriteHandle or VmdkWriteHandle. In the case of + image service as the destination, an instance of ImageWriter task is + created which reads from the blocking queue and writes to the image + service. + + :param context: write context needed for the image service + :param timeout_secs: time in seconds to wait for the transfer to complete + :param read_file_handle: handle to read data from + :param max_data_size: maximum transfer size + :param write_file_handle: handle to write data to; if this is None, then + param image_service and param image_id should + be set. + :param image_service: image service handle + :param image_id: ID of the image in the image service + :param image_meta: image meta-data + :raises: ImageTransferException, ValueError + """ + + # Create the blocking queue + blocking_queue = BlockingQueue(BLOCKING_QUEUE_SIZE, max_data_size) + + # Create the image reader + reader = FileReadWriteTask(read_file_handle, blocking_queue) + + # Create the image writer + if write_file_handle: + # File or VMDK in VMware datastore is the image destination + writer = FileReadWriteTask(blocking_queue, write_file_handle) + elif image_service and image_id: + # Image service image is the destination + writer = ImageWriter(context, + blocking_queue, + image_service, + image_id, + image_meta) + else: + excep_msg = _("No image destination given.") + LOG.error(excep_msg) + raise ValueError(excep_msg) + + # Start the reader and writer + LOG.debug("Starting image transfer with reader: %(reader)s and writer: " + "%(writer)s", + {'reader': reader, + 'writer': writer}) + reader.start() + writer.start() + timer = timeout.Timeout(timeout_secs) + try: + # Wait for the reader and writer to complete + reader.wait() + writer.wait() + except (timeout.Timeout, exceptions.ImageTransferException) as excep: + excep_msg = (_("Error occurred during image transfer with reader: " + "%(reader)s and writer: %(writer)s") % + {'reader': reader, + 'writer': writer}) + LOG.exception(excep_msg) + reader.stop() + writer.stop() + + if isinstance(excep, exceptions.ImageTransferException): + raise + raise exceptions.ImageTransferException(excep_msg, excep) + finally: + timer.cancel() + read_file_handle.close() + if write_file_handle: + write_file_handle.close() + + +def download_image(image, image_meta, session, datastore, rel_path, + bypass=True, timeout_secs=7200): + """Transfer an image to a datastore. + + :param image: file-like iterator + :param image_meta: image metadata + :param session: VMwareAPISession object + :param datastore: Datastore object + :param rel_path: path where the file will be stored in the datastore + :param bypass: if set to True, bypass vCenter to download the image + :param timeout_secs: time in seconds to wait for the xfer to complete + """ + image_size = int(image_meta['size']) + method = 'PUT' + if bypass: + hosts = datastore.get_connected_hosts(session) + host = ds_obj.Datastore.choose_host(hosts) + host_name = session.invoke_api(vim_util, 'get_object_property', + session.vim, host, 'name') + ds_url = datastore.build_url(session._scheme, host_name, rel_path, + constants.ESX_DATACENTER_PATH) + cookie = ds_url.get_transfer_ticket(session, method) + conn = ds_url.connect(method, image_size, cookie) + else: + ds_url = datastore.build_url(session._scheme, session._host, rel_path) + cookie = '%s=%s' % (constants.SOAP_COOKIE_KEY, + session.vim.get_http_cookie().strip("\"")) + conn = ds_url.connect(method, image_size, cookie) + conn.write = conn.send + + read_handle = rw_handles.ImageReadHandle(image) + _start_transfer(None, timeout_secs, read_handle, image_size, + write_file_handle=conn) + + +def download_flat_image(context, timeout_secs, image_service, image_id, + **kwargs): + """Download flat image from the image service to VMware server. + + :param context: image service write context + :param timeout_secs: time in seconds to wait for the download to complete + :param image_service: image service handle + :param image_id: ID of the image to be downloaded + :param kwargs: keyword arguments to configure the destination + file write handle + :raises: VimConnectionException, ImageTransferException, ValueError + """ + LOG.debug("Downloading image: %s from image service as a flat file.", + image_id) + + # TODO(vbala) catch specific exceptions raised by download call + read_iter = image_service.download(context, image_id) + read_handle = rw_handles.ImageReadHandle(read_iter) + file_size = int(kwargs.get('image_size')) + write_handle = rw_handles.FileWriteHandle(kwargs.get('host'), + kwargs.get('port'), + kwargs.get('data_center_name'), + kwargs.get('datastore_name'), + kwargs.get('cookies'), + kwargs.get('file_path'), + file_size, + cacerts=kwargs.get('cacerts')) + _start_transfer(context, + timeout_secs, + read_handle, + file_size, + write_file_handle=write_handle) + LOG.debug("Downloaded image: %s from image service as a flat file.", + image_id) + + +def download_stream_optimized_data(context, timeout_secs, read_handle, + **kwargs): + """Download stream optimized data to VMware server. + + :param context: image service write context + :param timeout_secs: time in seconds to wait for the download to complete + :param read_handle: handle from which to read the image data + :param kwargs: keyword arguments to configure the destination + VMDK write handle + :returns: managed object reference of the VM created for import to VMware + server + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException, + ImageTransferException, ValueError + """ + file_size = int(kwargs.get('image_size')) + write_handle = rw_handles.VmdkWriteHandle(kwargs.get('session'), + kwargs.get('host'), + kwargs.get('port'), + kwargs.get('resource_pool'), + kwargs.get('vm_folder'), + kwargs.get('vm_import_spec'), + file_size) + _start_transfer(context, + timeout_secs, + read_handle, + file_size, + write_file_handle=write_handle) + return write_handle.get_imported_vm() + + +def download_stream_optimized_image(context, timeout_secs, image_service, + image_id, **kwargs): + """Download stream optimized image from image service to VMware server. + + :param context: image service write context + :param timeout_secs: time in seconds to wait for the download to complete + :param image_service: image service handle + :param image_id: ID of the image to be downloaded + :param kwargs: keyword arguments to configure the destination + VMDK write handle + :returns: managed object reference of the VM created for import to VMware + server + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException, + ImageTransferException, ValueError + """ + LOG.debug("Downloading image: %s from image service as a stream " + "optimized file.", + image_id) + + # TODO(vbala) catch specific exceptions raised by download call + read_iter = image_service.download(context, image_id) + read_handle = rw_handles.ImageReadHandle(read_iter) + imported_vm = download_stream_optimized_data(context, timeout_secs, + read_handle, **kwargs) + + LOG.debug("Downloaded image: %s from image service as a stream " + "optimized file.", + image_id) + return imported_vm + + +def copy_stream_optimized_disk( + context, timeout_secs, write_handle, **kwargs): + """Copy virtual disk from VMware server to the given write handle. + + :param context: context + :param timeout_secs: time in seconds to wait for the copy to complete + :param write_handle: copy destination + :param kwargs: keyword arguments to configure the source + VMDK read handle + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException, + ImageTransferException, ValueError + """ + vmdk_file_path = kwargs.get('vmdk_file_path') + LOG.debug("Copying virtual disk: %(vmdk_path)s to %(dest)s.", + {'vmdk_path': vmdk_file_path, + 'dest': write_handle.name}) + file_size = kwargs.get('vmdk_size') + read_handle = rw_handles.VmdkReadHandle(kwargs.get('session'), + kwargs.get('host'), + kwargs.get('port'), + kwargs.get('vm'), + kwargs.get('vmdk_file_path'), + file_size) + _start_transfer(context, timeout_secs, read_handle, file_size, + write_file_handle=write_handle) + LOG.debug("Downloaded virtual disk: %s.", vmdk_file_path) + + +def upload_image(context, timeout_secs, image_service, image_id, owner_id, + **kwargs): + """Upload the VM's disk file to image service. + + :param context: image service write context + :param timeout_secs: time in seconds to wait for the upload to complete + :param image_service: image service handle + :param image_id: upload destination image ID + :param kwargs: keyword arguments to configure the source + VMDK read handle + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException, + ImageTransferException, ValueError + """ + + LOG.debug("Uploading to image: %s.", image_id) + file_size = kwargs.get('vmdk_size') + read_handle = rw_handles.VmdkReadHandle(kwargs.get('session'), + kwargs.get('host'), + kwargs.get('port'), + kwargs.get('vm'), + kwargs.get('vmdk_file_path'), + file_size) + + # Set the image properties. It is important to set the 'size' to 0. + # Otherwise, the image service client will use the VM's disk capacity + # which will not be the image size after upload, since it is converted + # to a stream-optimized sparse disk. + image_metadata = {'disk_format': 'vmdk', + 'is_public': kwargs.get('is_public'), + 'name': kwargs.get('image_name'), + 'status': 'active', + 'container_format': 'bare', + 'size': 0, + 'properties': {'vmware_image_version': + kwargs.get('image_version'), + 'vmware_disktype': 'streamOptimized', + 'owner_id': owner_id}} + + # Passing 0 as the file size since data size to be transferred cannot be + # predetermined. + _start_transfer(context, + timeout_secs, + read_handle, + 0, + image_service=image_service, + image_id=image_id, + image_meta=image_metadata) + LOG.debug("Uploaded image: %s.", image_id) diff --git a/oslo_vmware/objects/__init__.py b/oslo_vmware/objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oslo_vmware/objects/datacenter.py b/oslo_vmware/objects/datacenter.py new file mode 100644 index 00000000..76570422 --- /dev/null +++ b/oslo_vmware/objects/datacenter.py @@ -0,0 +1,27 @@ +# Copyright (c) 2014 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_vmware._i18n import _ + + +class Datacenter(object): + + def __init__(self, ref, name): + """Datacenter object holds ref and name together for convenience.""" + if name is None: + raise ValueError(_("Datacenter name cannot be None")) + if ref is None: + raise ValueError(_("Datacenter reference cannot be None")) + self.ref = ref + self.name = name diff --git a/oslo_vmware/objects/datastore.py b/oslo_vmware/objects/datastore.py new file mode 100644 index 00000000..234bf744 --- /dev/null +++ b/oslo_vmware/objects/datastore.py @@ -0,0 +1,318 @@ +# Copyright (c) 2014 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import posixpath +import random + +import six.moves.http_client as httplib +import six.moves.urllib.parse as urlparse + +from oslo_vmware._i18n import _ +from oslo_vmware import constants +from oslo_vmware import exceptions +from oslo_vmware import vim_util + +LOG = logging.getLogger(__name__) + + +class Datastore(object): + + def __init__(self, ref, name, capacity=None, freespace=None, + type=None, datacenter=None): + """Datastore object holds ref and name together for convenience. + + :param ref: a vSphere reference to a datastore + :param name: vSphere unique name for this datastore + :param capacity: (optional) capacity in bytes of this datastore + :param freespace: (optional) free space in bytes of datastore + :param type: (optional) datastore type + :param datacenter: (optional) oslo_vmware Datacenter object + """ + if name is None: + raise ValueError(_("Datastore name cannot be None")) + if ref is None: + raise ValueError(_("Datastore reference cannot be None")) + if freespace is not None and capacity is None: + raise ValueError(_("Invalid capacity")) + if capacity is not None and freespace is not None: + if capacity < freespace: + raise ValueError(_("Capacity is smaller than free space")) + + self.ref = ref + self.name = name + self.capacity = capacity + self.freespace = freespace + self.type = type + self.datacenter = datacenter + + def build_path(self, *paths): + """Constructs and returns a DatastorePath. + + :param paths: list of path components, for constructing a path relative + to the root directory of the datastore + :return: a DatastorePath object + """ + return DatastorePath(self.name, *paths) + + def build_url(self, scheme, server, rel_path, datacenter_name=None): + """Constructs and returns a DatastoreURL. + + :param scheme: scheme of the URL (http, https). + :param server: hostname or ip + :param rel_path: relative path of the file on the datastore + :param datacenter_name: (optional) datacenter name + :return: a DatastoreURL object + """ + if self.datacenter is None and datacenter_name is None: + raise ValueError(_("datacenter must be set to build url")) + if datacenter_name is None: + datacenter_name = self.datacenter.name + return DatastoreURL(scheme, server, rel_path, datacenter_name, + self.name) + + def __str__(self): + return '[%s]' % self._name + + def get_summary(self, session): + """Get datastore summary. + + :param datastore: Reference to the datastore + :return: 'summary' property of the datastore + """ + return session.invoke_api(vim_util, 'get_object_property', + session.vim, self.ref, 'summary') + + def get_connected_hosts(self, session): + """Get a list of usable (accessible, mounted, read-writable) hosts where + the datastore is mounted. + + :param: session: session + :return: list of HostSystem managed object references + """ + hosts = [] + summary = self.get_summary(session) + if not summary.accessible: + return hosts + host_mounts = session.invoke_api(vim_util, 'get_object_property', + session.vim, self.ref, 'host') + if not hasattr(host_mounts, 'DatastoreHostMount'): + return hosts + for host_mount in host_mounts.DatastoreHostMount: + if self.is_datastore_mount_usable(host_mount.mountInfo): + hosts.append(host_mount.key) + return hosts + + @staticmethod + def is_datastore_mount_usable(mount_info): + """Check if a datastore is usable as per the given mount info. + + The datastore is considered to be usable for a host only if it is + writable, mounted and accessible. + + :param mount_info: HostMountInfo data object + :return: True if datastore is usable + """ + writable = mount_info.accessMode == 'readWrite' + mounted = getattr(mount_info, 'mounted', True) + accessible = getattr(mount_info, 'accessible', False) + + return writable and mounted and accessible + + @staticmethod + def choose_host(hosts): + i = random.randrange(0, len(hosts)) + return hosts[i] + + +class DatastorePath(object): + + """Class for representing a directory or file path in a vSphere datatore. + + This provides various helper methods to access components and useful + variants of the datastore path. + + Example usage: + + DatastorePath("datastore1", "_base/foo", "foo.vmdk") creates an + object that describes the "[datastore1] _base/foo/foo.vmdk" datastore + file path to a virtual disk. + + Note: + - Datastore path representations always uses forward slash as separator + (hence the use of the posixpath module). + - Datastore names are enclosed in square brackets. + - Path part of datastore path is relative to the root directory + of the datastore, and is always separated from the [ds_name] part with + a single space. + """ + + def __init__(self, datastore_name, *paths): + if datastore_name is None or datastore_name == '': + raise ValueError(_("Datastore name cannot be empty")) + self._datastore_name = datastore_name + self._rel_path = '' + if paths: + if None in paths: + raise ValueError(_("Path component cannot be None")) + self._rel_path = posixpath.join(*paths) + + def __str__(self): + """Full datastore path to the file or directory.""" + if self._rel_path != '': + return "[%s] %s" % (self._datastore_name, self.rel_path) + return "[%s]" % self._datastore_name + + @property + def datastore(self): + return self._datastore_name + + @property + def parent(self): + return DatastorePath(self.datastore, posixpath.dirname(self._rel_path)) + + @property + def basename(self): + return posixpath.basename(self._rel_path) + + @property + def dirname(self): + return posixpath.dirname(self._rel_path) + + @property + def rel_path(self): + return self._rel_path + + def join(self, *paths): + """Join one or more path components intelligently into a datastore path. + + If any component is an absolute path, all previous components are + thrown away, and joining continues. The return value is the + concatenation of the paths with exactly one slash ('/') inserted + between components, unless p is empty. + + :return: A datastore path + """ + if paths: + if None in paths: + raise ValueError(_("Path component cannot be None")) + return DatastorePath(self.datastore, self._rel_path, *paths) + return self + + def __eq__(self, other): + return (isinstance(other, DatastorePath) and + self._datastore_name == other._datastore_name and + self._rel_path == other._rel_path) + + @classmethod + def parse(cls, datastore_path): + """Constructs a DatastorePath object given a datastore path string.""" + if not datastore_path: + raise ValueError(_("Datastore path cannot be empty")) + + spl = datastore_path.split('[', 1)[1].split(']', 1) + path = "" + if len(spl) == 1: + datastore_name = spl[0] + else: + datastore_name, path = spl + return cls(datastore_name, path.strip()) + + +class DatastoreURL(object): + + """Class for representing a URL to HTTP access a file in a datastore. + + This provides various helper methods to access components and useful + variants of the datastore URL. + """ + + def __init__(self, scheme, server, path, datacenter_path, datastore_name): + self._scheme = scheme + self._server = server + self._path = path + self._datacenter_path = datacenter_path + self._datastore_name = datastore_name + params = {'dcPath': self._datacenter_path, + 'dsName': self._datastore_name} + self._query = urlparse.urlencode(params) + + @classmethod + def urlparse(cls, url): + scheme, server, path, params, query, fragment = urlparse.urlparse(url) + if not query: + path = path.split('?') + query = path[1] + path = path[0] + params = urlparse.parse_qs(query) + dc_path = params.get('dcPath') + if dc_path is not None and len(dc_path) > 0: + datacenter_path = dc_path[0] + ds_name = params.get('dsName') + if ds_name is not None and len(ds_name) > 0: + datastore_name = ds_name[0] + path = path[len('/folder'):] + return cls(scheme, server, path, datacenter_path, datastore_name) + + @property + def path(self): + return self._path.strip('/') + + @property + def datacenter_path(self): + return self._datacenter_path + + @property + def datastore_name(self): + return self._datastore_name + + def __str__(self): + return '%s://%s/folder/%s?%s' % (self._scheme, self._server, + self.path, self._query) + + def connect(self, method, content_length, cookie): + try: + if self._scheme == 'http': + conn = httplib.HTTPConnection(self._server) + elif self._scheme == 'https': + conn = httplib.HTTPSConnection(self._server) + else: + excep_msg = _("Invalid scheme: %s.") % self._scheme + LOG.error(excep_msg) + raise ValueError(excep_msg) + conn.putrequest(method, '/folder/%s?%s' % (self.path, self._query)) + conn.putheader('User-Agent', constants.USER_AGENT) + conn.putheader('Content-Length', content_length) + conn.putheader('Cookie', cookie) + conn.endheaders() + LOG.debug("Created HTTP connection to transfer the file with " + "URL = %s.", str(self)) + return conn + except (httplib.InvalidURL, httplib.CannotSendRequest, + httplib.CannotSendHeader) as excep: + excep_msg = _("Error occurred while creating HTTP connection " + "to write to file with URL = %s.") % str(self) + LOG.exception(excep_msg) + raise exceptions.VimConnectionException(excep_msg, excep) + + def get_transfer_ticket(self, session, method): + client_factory = session.vim.client.factory + spec = vim_util.get_http_service_request_spec(client_factory, method, + str(self)) + ticket = session.invoke_api( + session.vim, + 'AcquireGenericServiceTicket', + session.vim.service_content.sessionManager, + spec=spec) + return '%s="%s"' % (constants.CGI_COOKIE_KEY, ticket.id) diff --git a/oslo_vmware/pbm.py b/oslo_vmware/pbm.py new file mode 100644 index 00000000..99abb617 --- /dev/null +++ b/oslo_vmware/pbm.py @@ -0,0 +1,200 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +VMware PBM service client and PBM related utility methods + +PBM is used for policy based placement in VMware datastores. +Refer http://goo.gl/GR2o6U for more details. +""" + +import logging +import os + +import six.moves.urllib.parse as urlparse +import six.moves.urllib.request as urllib +import suds.sax.element as element + +from oslo_vmware._i18n import _LW +from oslo_vmware import service +from oslo_vmware import vim_util + + +SERVICE_TYPE = 'PbmServiceInstance' + +LOG = logging.getLogger(__name__) + + +class Pbm(service.Service): + """Service class that provides access to the Storage Policy API.""" + + def __init__(self, protocol='https', host='localhost', port=443, + wsdl_url=None, cacert=None, insecure=True): + """Constructs a PBM service client object. + + :param protocol: http or https + :param host: server IP address or host name + :param port: port for connection + :param wsdl_url: PBM WSDL url + :param cacert: Specify a CA bundle file to use in verifying a + TLS (https) server certificate. + :param insecure: Verify HTTPS connections using system certificates, + used only if cacert is not specified + """ + base_url = service.Service.build_base_url(protocol, host, port) + soap_url = base_url + '/pbm' + super(Pbm, self).__init__(wsdl_url, soap_url, cacert, insecure) + + def set_soap_cookie(self, cookie): + """Set the specified vCenter session cookie in the SOAP header + + :param cookie: cookie to set + """ + elem = element.Element('vcSessionCookie').setText(cookie) + self.client.set_options(soapheaders=elem) + + def retrieve_service_content(self): + ref = vim_util.get_moref(service.SERVICE_INSTANCE, SERVICE_TYPE) + return self.PbmRetrieveServiceContent(ref) + + def __repr__(self): + return "PBM Object" + + def __str__(self): + return "PBM Object" + + +def get_all_profiles(session): + """Get all the profiles defined in VC server. + + :returns: PbmProfile data objects + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + LOG.debug("Fetching all the profiles defined in VC server.") + + pbm = session.pbm + profile_manager = pbm.service_content.profileManager + res_type = pbm.client.factory.create('ns0:PbmProfileResourceType') + res_type.resourceType = 'STORAGE' + profiles = [] + profile_ids = session.invoke_api(pbm, + 'PbmQueryProfile', + profile_manager, + resourceType=res_type) + LOG.debug("Fetched profile IDs: %s.", profile_ids) + if profile_ids: + profiles = session.invoke_api(pbm, + 'PbmRetrieveContent', + profile_manager, + profileIds=profile_ids) + return profiles + + +def get_profile_id_by_name(session, profile_name): + """Get the profile UUID corresponding to the given profile name. + + :param profile_name: profile name whose UUID needs to be retrieved + :returns: profile UUID string or None if profile not found + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + LOG.debug("Retrieving profile ID for profile: %s.", profile_name) + for profile in get_all_profiles(session): + if profile.name == profile_name: + profile_id = profile.profileId + LOG.debug("Retrieved profile ID: %(id)s for profile: %(name)s.", + {'id': profile_id, + 'name': profile_name}) + return profile_id + return None + + +def filter_hubs_by_profile(session, hubs, profile_id): + """Filter and return hubs that match the given profile. + + :param hubs: PbmPlacementHub morefs + :param profile_id: profile ID + :returns: subset of hubs that match the given profile + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + LOG.debug("Filtering hubs: %(hubs)s that match profile: %(profile)s.", + {'hubs': hubs, + 'profile': profile_id}) + + pbm = session.pbm + placement_solver = pbm.service_content.placementSolver + filtered_hubs = session.invoke_api(pbm, + 'PbmQueryMatchingHub', + placement_solver, + hubsToSearch=hubs, + profile=profile_id) + LOG.debug("Filtered hubs: %s", filtered_hubs) + return filtered_hubs + + +def convert_datastores_to_hubs(pbm_client_factory, datastores): + """Convert given datastore morefs to PbmPlacementHub morefs. + + :param pbm_client_factory: Factory to create PBM API input specs + :param datastores: list of datastore morefs + :returns: list of PbmPlacementHub morefs + """ + hubs = [] + for ds in datastores: + hub = pbm_client_factory.create('ns0:PbmPlacementHub') + hub.hubId = ds.value + hub.hubType = 'Datastore' + hubs.append(hub) + return hubs + + +def filter_datastores_by_hubs(hubs, datastores): + """Get filtered subset of datastores corresponding to the given hub list. + + :param hubs: list of PbmPlacementHub morefs + :param datastores: all candidate datastores + :returns: subset of datastores corresponding to the given hub list + """ + filtered_dss = [] + hub_ids = [hub.hubId for hub in hubs] + for ds in datastores: + if ds.value in hub_ids: + filtered_dss.append(ds) + return filtered_dss + + +def get_pbm_wsdl_location(vc_version): + """Return PBM WSDL file location corresponding to VC version. + + :param vc_version: a dot-separated version string. For example, "1.2". + :return: the pbm wsdl file location. + """ + if not vc_version: + return + ver = vc_version.split('.') + major_minor = ver[0] + if len(ver) >= 2: + major_minor = '%s.%s' % (major_minor, ver[1]) + curr_dir = os.path.abspath(os.path.dirname(__file__)) + pbm_service_wsdl = os.path.join(curr_dir, 'wsdl', major_minor, + 'pbmService.wsdl') + if not os.path.exists(pbm_service_wsdl): + LOG.warn(_LW("PBM WSDL file %s not found."), pbm_service_wsdl) + return + pbm_wsdl = urlparse.urljoin('file:', urllib.pathname2url(pbm_service_wsdl)) + LOG.debug("Using PBM WSDL location: %s.", pbm_wsdl) + return pbm_wsdl diff --git a/oslo_vmware/rw_handles.py b/oslo_vmware/rw_handles.py new file mode 100644 index 00000000..478c3a50 --- /dev/null +++ b/oslo_vmware/rw_handles.py @@ -0,0 +1,632 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Classes defining read and write handles for image transfer. + +This module defines various classes for reading and writing files including +VMDK files in VMware servers. It also contains a class to read images from +glance server. +""" + +import logging +import ssl + +import requests +import six +import six.moves.urllib.parse as urlparse +from urllib3 import connection as httplib + +from oslo.utils import excutils +from oslo.utils import netutils +from oslo_vmware._i18n import _, _LE, _LW +from oslo_vmware import exceptions +from oslo_vmware import vim_util + + +LOG = logging.getLogger(__name__) + +MIN_PROGRESS_DIFF_TO_LOG = 25 +READ_CHUNKSIZE = 65536 +USER_AGENT = 'OpenStack-ESX-Adapter' + + +class FileHandle(object): + """Base class for VMware server file (including VMDK) access over HTTP. + + This class wraps a backing file handle and provides utility methods + for various sub-classes. + """ + + def __init__(self, file_handle): + """Initializes the file handle. + + :param file_handle: backing file handle + """ + self._eof = False + self._file_handle = file_handle + self._last_logged_progress = 0 + + def _create_read_connection(self, url, cookies=None, cacerts=False): + LOG.debug("Opening URL: %s for reading.", url) + try: + headers = {'User-Agent': USER_AGENT} + if cookies: + headers.update({'Cookie': + self._build_vim_cookie_header(cookies)}) + response = requests.get(url, headers=headers, stream=True, + verify=cacerts) + return response.raw + except Exception as excep: + # TODO(vbala) We need to catch and raise specific exceptions + # related to connection problems, invalid request and invalid + # arguments. + excep_msg = _("Error occurred while opening URL: %s for " + "reading.") % url + LOG.exception(excep_msg) + raise exceptions.VimException(excep_msg, excep) + + def _create_write_connection(self, url, + file_size=None, + cookies=None, + overwrite=None, + content_type=None, + cacerts=False): + """Create HTTP connection to write to VMDK file.""" + LOG.debug("Creating HTTP connection to write to file with " + "size = %(file_size)d and URL = %(url)s.", + {'file_size': file_size, + 'url': url}) + _urlparse = urlparse.urlparse(url) + scheme, netloc, path, params, query, fragment = _urlparse + + try: + if scheme == 'http': + conn = httplib.HTTPConnection(netloc) + elif scheme == 'https': + conn = httplib.HTTPSConnection(netloc) + cert_reqs = None + + # cacerts can be either True or False or contain + # actual certificates. If it is a boolean, then + # we need to set cert_reqs and clear the cacerts + if isinstance(cacerts, bool): + if cacerts: + cert_reqs = ssl.CERT_REQUIRED + else: + cert_reqs = ssl.CERT_NONE + cacerts = None + + conn.set_cert(ca_certs=cacerts, cert_reqs=cert_reqs) + else: + excep_msg = _("Invalid scheme: %s.") % scheme + LOG.error(excep_msg) + raise ValueError(excep_msg) + + if query: + path = path + '?' + query + + headers = {'User-Agent': USER_AGENT} + if file_size: + headers.update({'Content-Length': str(file_size)}) + if overwrite: + headers.update({'Overwrite': overwrite}) + if cookies: + headers.update({'Cookie': + self._build_vim_cookie_header(cookies)}) + if content_type: + headers.update({'Content-Type': content_type}) + + conn.putrequest('PUT', path) + for key, value in six.iteritems(headers): + conn.putheader(key, value) + conn.endheaders() + return conn + except requests.RequestException as excep: + excep_msg = _("Error occurred while creating HTTP connection " + "to write to VMDK file with URL = %s.") % url + LOG.exception(excep_msg) + raise exceptions.VimConnectionException(excep_msg, excep) + + def close(self): + """Close the file handle.""" + try: + self._file_handle.close() + except Exception: + LOG.warn(_LW("Error occurred while closing the file handle"), + exc_info=True) + + def _build_vim_cookie_header(self, vim_cookies): + """Build ESX host session cookie header.""" + cookie_header = "" + for vim_cookie in vim_cookies: + cookie_header = vim_cookie.name + '=' + vim_cookie.value + break + return cookie_header + + def write(self, data): + """Write data to the file. + + :param data: data to be written + :raises: NotImplementedError + """ + raise NotImplementedError() + + def read(self, chunk_size): + """Read a chunk of data. + + :param chunk_size: read chunk size + :raises: NotImplementedError + """ + raise NotImplementedError() + + def get_size(self): + """Get size of the file to be read. + + :raises: NotImplementedError + """ + raise NotImplementedError() + + def _get_soap_url(self, scheme, host, port): + """Returns the IPv4/v6 compatible SOAP URL for the given host.""" + if netutils.is_valid_ipv6(host): + return '%s://[%s]:%d' % (scheme, host, port) + return '%s://%s:%d' % (scheme, host, port) + + def _fix_esx_url(self, url, host, port): + """Fix netloc in the case of an ESX host. + + In the case of an ESX host, the netloc is set to '*' in the URL + returned in HttpNfcLeaseInfo. It should be replaced with host name + or IP address. + """ + urlp = urlparse.urlparse(url) + if urlp.netloc == '*': + scheme, netloc, path, params, query, fragment = urlp + if netutils.is_valid_ipv6(host): + netloc = '[%s]:%d' % (host, port) + else: + netloc = "%s:%d" % (host, port) + url = urlparse.urlunparse((scheme, + netloc, + path, + params, + query, + fragment)) + return url + + def _find_vmdk_url(self, lease_info, host, port): + """Find the URL corresponding to a VMDK file in lease info.""" + url = None + for deviceUrl in lease_info.deviceUrl: + if deviceUrl.disk: + url = self._fix_esx_url(deviceUrl.url, host, port) + break + if not url: + excep_msg = _("Could not retrieve VMDK URL from lease info.") + LOG.error(excep_msg) + raise exceptions.VimException(excep_msg) + LOG.debug("Found VMDK URL: %s from lease info.", url) + return url + + def _log_progress(self, progress): + """Log data transfer progress.""" + if (progress == 100 or (progress - self._last_logged_progress >= + MIN_PROGRESS_DIFF_TO_LOG)): + LOG.debug("Data transfer progress is %d%%.", progress) + self._last_logged_progress = progress + + +class FileWriteHandle(FileHandle): + """Write handle for a file in VMware server.""" + + def __init__(self, host, port, data_center_name, datastore_name, cookies, + file_path, file_size, scheme='https', cacerts=False): + """Initializes the write handle with given parameters. + + :param host: ESX/VC server IP address or host name + :param port: port for connection + :param data_center_name: name of the data center in the case of a VC + server + :param datastore_name: name of the datastore where the file is stored + :param cookies: cookies to build the vim cookie header + :param file_path: datastore path where the file is written + :param file_size: size of the file in bytes + :param scheme: protocol-- http or https + :raises: VimConnectionException, ValueError + """ + soap_url = self._get_soap_url(scheme, host, port) + param_list = {'dcPath': data_center_name, 'dsName': datastore_name} + self._url = '%s/folder/%s' % (soap_url, file_path) + self._url = self._url + '?' + urlparse.urlencode(param_list) + + self._conn = self._create_write_connection(self._url, + file_size, + cookies=cookies, + cacerts=cacerts) + FileHandle.__init__(self, self._conn) + + def write(self, data): + """Write data to the file. + + :param data: data to be written + :raises: VimConnectionException, VimException + """ + try: + self._file_handle.send(data) + except requests.RequestException as excep: + excep_msg = _("Connection error occurred while writing data to" + " %s.") % self._url + LOG.exception(excep_msg) + raise exceptions.VimConnectionException(excep_msg, excep) + except Exception as excep: + # TODO(vbala) We need to catch and raise specific exceptions + # related to connection problems, invalid request and invalid + # arguments. + excep_msg = _("Error occurred while writing data to" + " %s.") % self._url + LOG.exception(excep_msg) + raise exceptions.VimException(excep_msg, excep) + + def close(self): + """Get the response and close the connection.""" + LOG.debug("Closing write handle for %s.", self._url) + try: + self._conn.getresponse() + except Exception: + LOG.warn(_LW("Error occurred while reading the HTTP response."), + exc_info=True) + super(FileWriteHandle, self).close() + + def __str__(self): + return "File write handle for %s" % self._url + + +class VmdkWriteHandle(FileHandle): + """VMDK write handle based on HttpNfcLease. + + This class creates a vApp in the specified resource pool and uploads the + virtual disk contents. + """ + + def __init__(self, session, host, port, rp_ref, vm_folder_ref, import_spec, + vmdk_size): + """Initializes the VMDK write handle with input parameters. + + :param session: valid API session to ESX/VC server + :param host: ESX/VC server IP address or host name + :param port: port for connection + :param rp_ref: resource pool into which the backing VM is imported + :param vm_folder_ref: VM folder in ESX/VC inventory to use as parent + of backing VM + :param import_spec: import specification of the backing VM + :param vmdk_size: size of the backing VM's VMDK file + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException, + ValueError + """ + self._session = session + self._vmdk_size = vmdk_size + self._bytes_written = 0 + + # Get lease and its info for vApp import + self._lease = self._create_and_wait_for_lease(session, + rp_ref, + import_spec, + vm_folder_ref) + LOG.debug("Invoking VIM API for reading info of lease: %s.", + self._lease) + lease_info = session.invoke_api(vim_util, + 'get_object_property', + session.vim, + self._lease, + 'info') + + # Find VMDK URL where data is to be written + self._url = self._find_vmdk_url(lease_info, host, port) + self._vm_ref = lease_info.entity + + cookies = session.vim.client.options.transport.cookiejar + # Create HTTP connection to write to VMDK URL + octet_stream = 'binary/octet-stream' + self._conn = self._create_write_connection(self._url, + vmdk_size, + cookies=cookies, + overwrite='t', + content_type=octet_stream, + cacerts=session._cacert) + FileHandle.__init__(self, self._conn) + + def get_imported_vm(self): + """"Get managed object reference of the VM created for import.""" + return self._vm_ref + + def _create_and_wait_for_lease(self, session, rp_ref, import_spec, + vm_folder_ref): + """Create and wait for HttpNfcLease lease for vApp import.""" + LOG.debug("Creating HttpNfcLease lease for vApp import into resource" + " pool: %s.", + rp_ref) + lease = session.invoke_api(session.vim, + 'ImportVApp', + rp_ref, + spec=import_spec, + folder=vm_folder_ref) + LOG.debug("Lease: %(lease)s obtained for vApp import into resource" + " pool %(rp_ref)s.", + {'lease': lease, + 'rp_ref': rp_ref}) + session.wait_for_lease_ready(lease) + return lease + + def write(self, data): + """Write data to the file. + + :param data: data to be written + :raises: VimConnectionException, VimException + """ + try: + self._file_handle.send(data) + self._bytes_written += len(data) + except requests.RequestException as excep: + excep_msg = _("Connection error occurred while writing data to" + " %s.") % self._url + LOG.exception(excep_msg) + raise exceptions.VimConnectionException(excep_msg, excep) + except Exception as excep: + # TODO(vbala) We need to catch and raise specific exceptions + # related to connection problems, invalid request and invalid + # arguments. + excep_msg = _("Error occurred while writing data to" + " %s.") % self._url + LOG.exception(excep_msg) + raise exceptions.VimException(excep_msg, excep) + + # TODO(vbala) Move this method to FileHandle. + def update_progress(self): + """Updates progress to lease. + + This call back to the lease is essential to keep the lease alive + across long running write operations. + + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + progress = int(float(self._bytes_written) / self._vmdk_size * 100) + self._log_progress(progress) + + try: + self._session.invoke_api(self._session.vim, + 'HttpNfcLeaseProgress', + self._lease, + percent=progress) + except exceptions.VimException: + with excutils.save_and_reraise_exception(): + LOG.exception(_LE("Error occurred while updating the " + "write progress of VMDK file with " + "URL = %s."), + self._url) + + def close(self): + """Releases the lease and close the connection. + + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + LOG.debug("Getting lease state for %s.", self._url) + try: + state = self._session.invoke_api(vim_util, + 'get_object_property', + self._session.vim, + self._lease, + 'state') + LOG.debug("Lease for %(url)s is in state: %(state)s.", + {'url': self._url, + 'state': state}) + if state == 'ready': + LOG.debug("Releasing lease for %s.", self._url) + self._session.invoke_api(self._session.vim, + 'HttpNfcLeaseComplete', + self._lease) + else: + LOG.debug("Lease for %(url)s is in state: %(state)s; no " + "need to release.", + {'url': self._url, + 'state': state}) + except exceptions.VimException: + LOG.warn(_LW("Error occurred while releasing the lease for %s."), + self._url, + exc_info=True) + super(VmdkWriteHandle, self).close() + LOG.debug("Closed VMDK write handle for %s.", self._url) + + def __str__(self): + return "VMDK write handle for %s" % self._url + + +class VmdkReadHandle(FileHandle): + """VMDK read handle based on HttpNfcLease.""" + + def __init__(self, session, host, port, vm_ref, vmdk_path, + vmdk_size): + """Initializes the VMDK read handle with the given parameters. + + During the read (export) operation, the VMDK file is converted to a + stream-optimized sparse disk format. Therefore, the size of the VMDK + file read may be smaller than the actual VMDK size. + + :param session: valid api session to ESX/VC server + :param host: ESX/VC server IP address or host name + :param port: port for connection + :param vm_ref: managed object reference of the backing VM whose VMDK + is to be exported + :param vmdk_path: path of the VMDK file to be exported + :param vmdk_size: actual size of the VMDK file + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + self._session = session + self._vmdk_size = vmdk_size + self._bytes_read = 0 + + # Obtain lease for VM export + self._lease = self._create_and_wait_for_lease(session, vm_ref) + LOG.debug("Invoking VIM API for reading info of lease: %s.", + self._lease) + lease_info = session.invoke_api(vim_util, + 'get_object_property', + session.vim, + self._lease, + 'info') + + # find URL of the VMDK file to be read and open connection + self._url = self._find_vmdk_url(lease_info, host, port) + cookies = session.vim.client.options.transport.cookiejar + cacerts = session.vim.client.options.transport.verify + self._conn = self._create_read_connection(self._url, + cookies=cookies, + cacerts=cacerts) + FileHandle.__init__(self, self._conn) + + def _create_and_wait_for_lease(self, session, vm_ref): + """Create and wait for HttpNfcLease lease for VM export.""" + LOG.debug("Creating HttpNfcLease lease for exporting VM: %s.", + vm_ref) + lease = session.invoke_api(session.vim, 'ExportVm', vm_ref) + LOG.debug("Lease: %(lease)s obtained for exporting VM: %(vm_ref)s.", + {'lease': lease, + 'vm_ref': vm_ref}) + session.wait_for_lease_ready(lease) + return lease + + def read(self, chunk_size): + """Read a chunk of data from the VMDK file. + + :param chunk_size: size of read chunk + :returns: the data + :raises: VimException + """ + try: + data = self._file_handle.read(READ_CHUNKSIZE) + self._bytes_read += len(data) + return data + except Exception as excep: + # TODO(vbala) We need to catch and raise specific exceptions + # related to connection problems, invalid request and invalid + # arguments. + excep_msg = _("Error occurred while reading data from" + " %s.") % self._url + LOG.exception(excep_msg) + raise exceptions.VimException(excep_msg, excep) + + def update_progress(self): + """Updates progress to lease. + + This call back to the lease is essential to keep the lease alive + across long running read operations. + + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + progress = int(float(self._bytes_read) / self._vmdk_size * 100) + self._log_progress(progress) + + try: + self._session.invoke_api(self._session.vim, + 'HttpNfcLeaseProgress', + self._lease, + percent=progress) + except exceptions.VimException: + with excutils.save_and_reraise_exception(): + LOG.exception(_LE("Error occurred while updating the " + "read progress of VMDK file with URL = %s."), + self._url) + + def close(self): + """Releases the lease and close the connection. + + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + LOG.debug("Getting lease state for %s.", self._url) + try: + state = self._session.invoke_api(vim_util, + 'get_object_property', + self._session.vim, + self._lease, + 'state') + LOG.debug("Lease for %(url)s is in state: %(state)s.", + {'url': self._url, + 'state': state}) + if state == 'ready': + LOG.debug("Releasing lease for %s.", self._url) + self._session.invoke_api(self._session.vim, + 'HttpNfcLeaseComplete', + self._lease) + else: + LOG.debug("Lease for %(url)s is in state: %(state)s; no " + "need to release.", + {'url': self._url, + 'state': state}) + except exceptions.VimException: + LOG.warn(_LW("Error occurred while releasing the lease for %s."), + self._url, + exc_info=True) + raise + super(VmdkReadHandle, self).close() + LOG.debug("Closed VMDK read handle for %s.", self._url) + + def __str__(self): + return "VMDK read handle for %s" % self._url + + +class ImageReadHandle(object): + """Read handle for glance images.""" + + def __init__(self, glance_read_iter): + """Initializes the read handle with given parameters. + + :param glance_read_iter: iterator to read data from glance image + """ + self._glance_read_iter = glance_read_iter + self._iter = self.get_next() + + def read(self, chunk_size): + """Read an item from the image data iterator. + + The input chunk size is ignored since the client ImageBodyIterator + uses its own chunk size. + """ + try: + data = next(self._iter) + return data + except StopIteration: + LOG.debug("Completed reading data from the image iterator.") + return "" + + def get_next(self): + """Get the next item from the image iterator.""" + for data in self._glance_read_iter: + yield data + + def close(self): + """Close the read handle. + + This is a NOP. + """ + pass + + def __str__(self): + return "Image read handle" diff --git a/oslo_vmware/service.py b/oslo_vmware/service.py new file mode 100644 index 00000000..5e45f7a8 --- /dev/null +++ b/oslo_vmware/service.py @@ -0,0 +1,357 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Common classes that provide access to vSphere services. +""" + +import logging +import os + +import netaddr +import requests +import six +import six.moves.http_client as httplib +import suds +from suds import cache +from suds import client +from suds import plugin +from suds import transport + +from oslo.utils import timeutils +from oslo_vmware._i18n import _ +from oslo_vmware import exceptions +from oslo_vmware import vim_util + +CACHE_TIMEOUT = 60 * 60 # One hour cache timeout +ADDRESS_IN_USE_ERROR = 'Address already in use' +CONN_ABORT_ERROR = 'Software caused connection abort' +RESP_NOT_XML_ERROR = 'Response is "text/html", not "text/xml"' + +SERVICE_INSTANCE = 'ServiceInstance' + +LOG = logging.getLogger(__name__) + + +class ServiceMessagePlugin(plugin.MessagePlugin): + """Suds plug-in handling some special cases while calling VI SDK.""" + + def add_attribute_for_value(self, node): + """Helper to handle AnyType. + + Suds does not handle AnyType properly. But VI SDK requires type + attribute to be set when AnyType is used. + + :param node: XML value node + """ + if node.name == 'value': + node.set('xsi:type', 'xsd:string') + + def marshalled(self, context): + """Modifies the envelope document before it is sent. + + This method provides the plug-in with the opportunity to prune empty + nodes and fix nodes before sending it to the server. + + :param context: send context + """ + # Suds builds the entire request object based on the WSDL schema. + # VI SDK throws server errors if optional SOAP nodes are sent + # without values; e.g., as opposed to test. + context.envelope.prune() + context.envelope.walk(self.add_attribute_for_value) + + +class Response(six.BytesIO): + """Response with an input stream as source.""" + + def __init__(self, stream, status=200, headers=None): + self.status = status + self.headers = headers or {} + self.reason = requests.status_codes._codes.get( + status, [''])[0].upper().replace('_', ' ') + six.BytesIO.__init__(self, stream) + + @property + def _original_response(self): + return self + + @property + def msg(self): + return self + + def read(self, chunk_size, **kwargs): + return six.BytesIO.read(self, chunk_size) + + def info(self): + return self + + def get_all(self, name, default): + result = self.headers.get(name) + if not result: + return default + return [result] + + def getheaders(self, name): + return self.get_all(name, []) + + def release_conn(self): + self.close() + + +class LocalFileAdapter(requests.adapters.HTTPAdapter): + """Transport adapter for local files. + + See http://stackoverflow.com/a/22989322 + """ + + def _build_response_from_file(self, request): + file_path = request.url[7:] + with open(file_path, 'r') as f: + buff = bytearray(os.path.getsize(file_path)) + f.readinto(buff) + resp = Response(buff) + return self.build_response(request, resp) + + def send(self, request, stream=False, timeout=None, + verify=True, cert=None, proxies=None): + return self._build_response_from_file(request) + + +class RequestsTransport(transport.Transport): + def __init__(self, cacert=None, insecure=True): + transport.Transport.__init__(self) + # insecure flag is used only if cacert is not + # specified. + self.verify = cacert if cacert else not insecure + self.session = requests.Session() + self.session.mount('file:///', LocalFileAdapter()) + self.cookiejar = self.session.cookies + + def open(self, request): + resp = self.session.get(request.url, verify=self.verify) + return six.StringIO(resp.content) + + def send(self, request): + resp = self.session.post(request.url, + data=request.message, + headers=request.headers, + verify=self.verify) + return transport.Reply(resp.status_code, resp.headers, resp.content) + + +class MemoryCache(cache.ObjectCache): + def __init__(self): + self._cache = {} + + def get(self, key): + """Retrieves the value for a key or None.""" + now = timeutils.utcnow_ts() + for k in list(self._cache): + (timeout, _value) = self._cache[k] + if timeout and now >= timeout: + del self._cache[k] + + return self._cache.get(key, (0, None))[1] + + def put(self, key, value, time=CACHE_TIMEOUT): + """Sets the value for a key.""" + timeout = 0 + if time != 0: + timeout = timeutils.utcnow_ts() + time + self._cache[key] = (timeout, value) + return True + + +_CACHE = MemoryCache() + + +class Service(object): + """Base class containing common functionality for invoking vSphere + services + """ + + def __init__(self, wsdl_url=None, soap_url=None, + cacert=None, insecure=True): + self.wsdl_url = wsdl_url + self.soap_url = soap_url + LOG.debug("Creating suds client with soap_url='%s' and wsdl_url='%s'", + self.soap_url, self.wsdl_url) + transport = RequestsTransport(cacert, insecure) + self.client = client.Client(self.wsdl_url, + transport=transport, + location=self.soap_url, + plugins=[ServiceMessagePlugin()], + cache=_CACHE) + self._service_content = None + + @staticmethod + def build_base_url(protocol, host, port): + proto_str = '%s://' % protocol + host_str = '[%s]' % host if netaddr.valid_ipv6(host) else host + port_str = '' if port is None else ':%d' % port + return proto_str + host_str + port_str + + @staticmethod + def _retrieve_properties_ex_fault_checker(response): + """Checks the RetrievePropertiesEx API response for errors. + + Certain faults are sent in the SOAP body as a property of missingSet. + This method raises VimFaultException when a fault is found in the + response. + + :param response: response from RetrievePropertiesEx API call + :raises: VimFaultException + """ + fault_list = [] + details = {} + if not response: + # This is the case when the session has timed out. ESX SOAP + # server sends an empty RetrievePropertiesExResponse. Normally + # missingSet in the response objects has the specifics about + # the error, but that's not the case with a timed out idle + # session. It is as bad as a terminated session for we cannot + # use the session. Therefore setting fault to NotAuthenticated + # fault. + LOG.debug("RetrievePropertiesEx API response is empty; setting " + "fault to %s.", + exceptions.NOT_AUTHENTICATED) + fault_list = [exceptions.NOT_AUTHENTICATED] + else: + for obj_cont in response.objects: + if hasattr(obj_cont, 'missingSet'): + for missing_elem in obj_cont.missingSet: + f_type = missing_elem.fault.fault + f_name = f_type.__class__.__name__ + fault_list.append(f_name) + if f_name == exceptions.NO_PERMISSION: + details['object'] = f_type.object.value + details['privilegeId'] = f_type.privilegeId + + if fault_list: + fault_string = _("Error occurred while calling " + "RetrievePropertiesEx.") + raise exceptions.VimFaultException(fault_list, + fault_string, + details=details) + + @property + def service_content(self): + if self._service_content is None: + self._service_content = self.retrieve_service_content() + return self._service_content + + def get_http_cookie(self): + """Return the vCenter session cookie.""" + cookies = self.client.options.transport.cookiejar + for cookie in cookies: + if cookie.name.lower() == 'vmware_soap_session': + return cookie.value + + def __getattr__(self, attr_name): + """Returns the method to invoke API identified by param attr_name.""" + + def request_handler(managed_object, **kwargs): + """Handler for vSphere API calls. + + Invokes the API and parses the response for fault checking and + other errors. + + :param managed_object: managed object reference argument of the + API call + :param kwargs: keyword arguments of the API call + :returns: response of the API call + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + try: + if isinstance(managed_object, str): + # For strings, use string value for value and type + # of the managed object. + managed_object = vim_util.get_moref(managed_object, + managed_object) + if managed_object is None: + return + request = getattr(self.client.service, attr_name) + response = request(managed_object, **kwargs) + if (attr_name.lower() == 'retrievepropertiesex'): + Service._retrieve_properties_ex_fault_checker(response) + return response + except exceptions.VimFaultException: + # Catch the VimFaultException that is raised by the fault + # check of the SOAP response. + raise + + except suds.WebFault as excep: + fault_string = None + if excep.fault: + fault_string = excep.fault.faultstring + + doc = excep.document + detail = None + if doc is not None: + detail = doc.childAtPath('/detail') + if not detail: + # NOTE(arnaud): this is needed with VC 5.1 + detail = doc.childAtPath('/Envelope/Body/Fault/detail') + fault_list = [] + details = {} + if detail: + for fault in detail.getChildren(): + fault_list.append(fault.get("type")) + for child in fault.getChildren(): + details[child.name] = child.getText() + raise exceptions.VimFaultException(fault_list, fault_string, + excep, details) + + except AttributeError as excep: + raise exceptions.VimAttributeException( + _("No such SOAP method %s.") % attr_name, excep) + + except (httplib.CannotSendRequest, + httplib.ResponseNotReady, + httplib.CannotSendHeader) as excep: + raise exceptions.VimSessionOverLoadException( + _("httplib error in %s.") % attr_name, excep) + + except requests.RequestException as excep: + raise exceptions.VimConnectionException( + _("requests error in %s.") % attr_name, excep) + + except Exception as excep: + # TODO(vbala) should catch specific exceptions and raise + # appropriate VimExceptions. + + # Socket errors which need special handling; some of these + # might be caused by server API call overload. + if (six.text_type(excep).find(ADDRESS_IN_USE_ERROR) != -1 or + six.text_type(excep).find(CONN_ABORT_ERROR)) != -1: + raise exceptions.VimSessionOverLoadException( + _("Socket error in %s.") % attr_name, excep) + # Type error which needs special handling; it might be caused + # by server API call overload. + elif six.text_type(excep).find(RESP_NOT_XML_ERROR) != -1: + raise exceptions.VimSessionOverLoadException( + _("Type error in %s.") % attr_name, excep) + else: + raise exceptions.VimException( + _("Exception in %s.") % attr_name, excep) + return request_handler + + def __repr__(self): + return "vSphere object" + + def __str__(self): + return "vSphere object" diff --git a/oslo_vmware/tests/__init__.py b/oslo_vmware/tests/__init__.py new file mode 100644 index 00000000..06807476 --- /dev/null +++ b/oslo_vmware/tests/__init__.py @@ -0,0 +1,11 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/oslo_vmware/tests/base.py b/oslo_vmware/tests/base.py new file mode 100644 index 00000000..69e6a802 --- /dev/null +++ b/oslo_vmware/tests/base.py @@ -0,0 +1,53 @@ +# Copyright 2010-2011 OpenStack Foundation +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import fixtures +import testtools + +_TRUE_VALUES = ('true', '1', 'yes') + +# FIXME(dhellmann) Update this to use oslo.test library + + +class TestCase(testtools.TestCase): + + """Test case base class for all unit tests.""" + + def setUp(self): + """Run before each test method to initialize test environment.""" + + super(TestCase, self).setUp() + test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0) + try: + test_timeout = int(test_timeout) + except ValueError: + # If timeout value is invalid do not set a timeout. + test_timeout = 0 + if test_timeout > 0: + self.useFixture(fixtures.Timeout(test_timeout, gentle=True)) + + self.useFixture(fixtures.NestedTempfile()) + self.useFixture(fixtures.TempHomeDir()) + + if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES: + stdout = self.useFixture(fixtures.StringStream('stdout')).stream + self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout)) + if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES: + stderr = self.useFixture(fixtures.StringStream('stderr')).stream + self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr)) + + self.log_fixture = self.useFixture(fixtures.FakeLogger()) diff --git a/oslo_vmware/tests/objects/__init__.py b/oslo_vmware/tests/objects/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/oslo_vmware/tests/objects/test_datacenter.py b/oslo_vmware/tests/objects/test_datacenter.py new file mode 100644 index 00000000..c4d261ed --- /dev/null +++ b/oslo_vmware/tests/objects/test_datacenter.py @@ -0,0 +1,30 @@ +# Copyright (c) 2014 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from oslo_vmware.objects import datacenter +from oslo_vmware.tests import base + + +class DatacenterTestCase(base.TestCase): + + """Test the Datacenter object.""" + + def test_dc(self): + self.assertRaises(ValueError, datacenter.Datacenter, None, 'dc-1') + self.assertRaises(ValueError, datacenter.Datacenter, mock.Mock(), None) + dc = datacenter.Datacenter('ref', 'name') + self.assertEqual('ref', dc.ref) + self.assertEqual('name', dc.name) diff --git a/oslo_vmware/tests/objects/test_datastore.py b/oslo_vmware/tests/objects/test_datastore.py new file mode 100644 index 00000000..cb36ecc4 --- /dev/null +++ b/oslo_vmware/tests/objects/test_datastore.py @@ -0,0 +1,384 @@ +# Copyright (c) 2014 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import six.moves.urllib.parse as urlparse + +from oslo.utils import units +from oslo_vmware import constants +from oslo_vmware.objects import datastore +from oslo_vmware.tests import base +from oslo_vmware import vim_util + + +class HostMount(object): + + def __init__(self, key, mountInfo): + self.key = key + self.mountInfo = mountInfo + + +class MountInfo(object): + + def __init__(self, accessMode, mounted, accessible): + self.accessMode = accessMode + self.mounted = mounted + self.accessible = accessible + + +class DatastoreTestCase(base.TestCase): + + """Test the Datastore object.""" + + def test_ds(self): + ds = datastore.Datastore( + "fake_ref", "ds_name", 2 * units.Gi, 1 * units.Gi) + self.assertEqual('ds_name', ds.name) + self.assertEqual('fake_ref', ds.ref) + self.assertEqual(2 * units.Gi, ds.capacity) + self.assertEqual(1 * units.Gi, ds.freespace) + + def test_ds_invalid_space(self): + self.assertRaises(ValueError, datastore.Datastore, + "fake_ref", "ds_name", 1 * units.Gi, 2 * units.Gi) + self.assertRaises(ValueError, datastore.Datastore, + "fake_ref", "ds_name", None, 2 * units.Gi) + + def test_ds_no_capacity_no_freespace(self): + ds = datastore.Datastore("fake_ref", "ds_name") + self.assertIsNone(ds.capacity) + self.assertIsNone(ds.freespace) + + def test_ds_invalid(self): + self.assertRaises(ValueError, datastore.Datastore, None, "ds_name") + self.assertRaises(ValueError, datastore.Datastore, "fake_ref", None) + + def test_build_path(self): + ds = datastore.Datastore("fake_ref", "ds_name") + ds_path = ds.build_path("some_dir", "foo.vmdk") + self.assertEqual('[ds_name] some_dir/foo.vmdk', str(ds_path)) + + def test_build_url(self): + ds = datastore.Datastore("fake_ref", "ds_name") + path = 'images/ubuntu.vmdk' + self.assertRaises(ValueError, ds.build_url, 'https', '10.0.0.2', path) + ds.datacenter = mock.Mock() + ds.datacenter.name = "dc_path" + ds_url = ds.build_url('https', '10.0.0.2', path) + self.assertEqual(ds_url.datastore_name, "ds_name") + self.assertEqual(ds_url.datacenter_path, "dc_path") + self.assertEqual(ds_url.path, path) + + def test_get_summary(self): + ds_ref = vim_util.get_moref('ds-0', 'Datastore') + ds = datastore.Datastore(ds_ref, 'ds-name') + summary = mock.sentinel.summary + session = mock.Mock() + session.invoke_api = mock.Mock() + session.invoke_api.return_value = summary + ret = ds.get_summary(session) + self.assertEqual(summary, ret) + session.invoke_api.assert_called_once_with(vim_util, + 'get_object_property', + session.vim, + ds.ref, 'summary') + + def test_get_connected_hosts(self): + session = mock.Mock() + ds_ref = vim_util.get_moref('ds-0', 'Datastore') + ds = datastore.Datastore(ds_ref, 'ds-name') + ds.get_summary = mock.Mock() + ds.get_summary.return_value.accessible = False + self.assertEqual([], ds.get_connected_hosts(session)) + ds.get_summary.return_value.accessible = True + m1 = HostMount("m1", MountInfo('readWrite', True, True)) + m2 = HostMount("m2", MountInfo('read', True, True)) + m3 = HostMount("m3", MountInfo('readWrite', False, True)) + m4 = HostMount("m4", MountInfo('readWrite', True, False)) + ds.get_summary.assert_called_once_with(session) + + class Prop(object): + DatastoreHostMount = [m1, m2, m3, m4] + session.invoke_api = mock.Mock() + session.invoke_api.return_value = Prop() + hosts = ds.get_connected_hosts(session) + self.assertEqual(1, len(hosts)) + self.assertEqual("m1", hosts.pop()) + + def test_is_datastore_mount_usable(self): + m = MountInfo('readWrite', True, True) + self.assertTrue(datastore.Datastore.is_datastore_mount_usable(m)) + m = MountInfo('read', True, True) + self.assertFalse(datastore.Datastore.is_datastore_mount_usable(m)) + m = MountInfo('readWrite', False, True) + self.assertFalse(datastore.Datastore.is_datastore_mount_usable(m)) + m = MountInfo('readWrite', True, False) + self.assertFalse(datastore.Datastore.is_datastore_mount_usable(m)) + m = MountInfo('readWrite', False, False) + self.assertFalse(datastore.Datastore.is_datastore_mount_usable(m)) + m = MountInfo('readWrite', None, None) + self.assertFalse(datastore.Datastore.is_datastore_mount_usable(m)) + m = MountInfo('readWrite', None, True) + self.assertFalse(datastore.Datastore.is_datastore_mount_usable(m)) + + +class DatastorePathTestCase(base.TestCase): + + """Test the DatastorePath object.""" + + def test_ds_path(self): + p = datastore.DatastorePath('dsname', 'a/b/c', 'file.iso') + self.assertEqual('[dsname] a/b/c/file.iso', str(p)) + self.assertEqual('a/b/c/file.iso', p.rel_path) + self.assertEqual('a/b/c', p.parent.rel_path) + self.assertEqual('[dsname] a/b/c', str(p.parent)) + self.assertEqual('dsname', p.datastore) + self.assertEqual('file.iso', p.basename) + self.assertEqual('a/b/c', p.dirname) + + def test_ds_path_no_ds_name(self): + bad_args = [ + ('', ['a/b/c', 'file.iso']), + (None, ['a/b/c', 'file.iso'])] + for t in bad_args: + self.assertRaises( + ValueError, datastore.DatastorePath, + t[0], *t[1]) + + def test_ds_path_invalid_path_components(self): + bad_args = [ + ('dsname', [None]), + ('dsname', ['', None]), + ('dsname', ['a', None]), + ('dsname', ['a', None, 'b']), + ('dsname', [None, '']), + ('dsname', [None, 'b'])] + + for t in bad_args: + self.assertRaises( + ValueError, datastore.DatastorePath, + t[0], *t[1]) + + def test_ds_path_no_subdir(self): + args = [ + ('dsname', ['', 'x.vmdk']), + ('dsname', ['x.vmdk'])] + + canonical_p = datastore.DatastorePath('dsname', 'x.vmdk') + self.assertEqual('[dsname] x.vmdk', str(canonical_p)) + self.assertEqual('', canonical_p.dirname) + self.assertEqual('x.vmdk', canonical_p.basename) + self.assertEqual('x.vmdk', canonical_p.rel_path) + for t in args: + p = datastore.DatastorePath(t[0], *t[1]) + self.assertEqual(str(canonical_p), str(p)) + + def test_ds_path_ds_only(self): + args = [ + ('dsname', []), + ('dsname', ['']), + ('dsname', ['', ''])] + + canonical_p = datastore.DatastorePath('dsname') + self.assertEqual('[dsname]', str(canonical_p)) + self.assertEqual('', canonical_p.rel_path) + self.assertEqual('', canonical_p.basename) + self.assertEqual('', canonical_p.dirname) + for t in args: + p = datastore.DatastorePath(t[0], *t[1]) + self.assertEqual(str(canonical_p), str(p)) + self.assertEqual(canonical_p.rel_path, p.rel_path) + + def test_ds_path_equivalence(self): + args = [ + ('dsname', ['a/b/c/', 'x.vmdk']), + ('dsname', ['a/', 'b/c/', 'x.vmdk']), + ('dsname', ['a', 'b', 'c', 'x.vmdk']), + ('dsname', ['a/b/c', 'x.vmdk'])] + + canonical_p = datastore.DatastorePath('dsname', 'a/b/c', 'x.vmdk') + for t in args: + p = datastore.DatastorePath(t[0], *t[1]) + self.assertEqual(str(canonical_p), str(p)) + self.assertEqual(canonical_p.datastore, p.datastore) + self.assertEqual(canonical_p.rel_path, p.rel_path) + self.assertEqual(str(canonical_p.parent), str(p.parent)) + + def test_ds_path_non_equivalence(self): + args = [ + # leading slash + ('dsname', ['/a', 'b', 'c', 'x.vmdk']), + ('dsname', ['/a/b/c/', 'x.vmdk']), + ('dsname', ['a/b/c', '/x.vmdk']), + # leading space + ('dsname', ['a/b/c/', ' x.vmdk']), + ('dsname', ['a/', ' b/c/', 'x.vmdk']), + ('dsname', [' a', 'b', 'c', 'x.vmdk']), + # trailing space + ('dsname', ['/a/b/c/', 'x.vmdk ']), + ('dsname', ['a/b/c/ ', 'x.vmdk'])] + + canonical_p = datastore.DatastorePath('dsname', 'a/b/c', 'x.vmdk') + for t in args: + p = datastore.DatastorePath(t[0], *t[1]) + self.assertNotEqual(str(canonical_p), str(p)) + + def test_equal(self): + a = datastore.DatastorePath('ds_name', 'a') + b = datastore.DatastorePath('ds_name', 'a') + self.assertEqual(a, b) + + def test_join(self): + p = datastore.DatastorePath('ds_name', 'a') + ds_path = p.join('b') + self.assertEqual('[ds_name] a/b', str(ds_path)) + + p = datastore.DatastorePath('ds_name', 'a') + ds_path = p.join() + bad_args = [ + [None], + ['', None], + ['a', None], + ['a', None, 'b']] + for arg in bad_args: + self.assertRaises(ValueError, p.join, *arg) + + def test_ds_path_parse(self): + p = datastore.DatastorePath.parse('[dsname]') + self.assertEqual('dsname', p.datastore) + self.assertEqual('', p.rel_path) + + p = datastore.DatastorePath.parse('[dsname] folder') + self.assertEqual('dsname', p.datastore) + self.assertEqual('folder', p.rel_path) + + p = datastore.DatastorePath.parse('[dsname] folder/file') + self.assertEqual('dsname', p.datastore) + self.assertEqual('folder/file', p.rel_path) + + for p in [None, '']: + self.assertRaises(ValueError, datastore.DatastorePath.parse, p) + + for p in ['bad path', '/a/b/c', 'a/b/c']: + self.assertRaises(IndexError, datastore.DatastorePath.parse, p) + + +class DatastoreURLTestCase(base.TestCase): + + """Test the DatastoreURL object.""" + + def test_path_strip(self): + scheme = 'https' + server = '13.37.73.31' + path = 'images/ubuntu-14.04.vmdk' + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + query = urlparse.urlencode(params) + url = datastore.DatastoreURL(scheme, server, path, dc_path, ds_name) + expected_url = '%s://%s/folder/%s?%s' % ( + scheme, server, path, query) + self.assertEqual(expected_url, str(url)) + + def test_path_lstrip(self): + scheme = 'https' + server = '13.37.73.31' + path = '/images/ubuntu-14.04.vmdk' + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + query = urlparse.urlencode(params) + url = datastore.DatastoreURL(scheme, server, path, dc_path, ds_name) + expected_url = '%s://%s/folder/%s?%s' % ( + scheme, server, path.lstrip('/'), query) + self.assertEqual(expected_url, str(url)) + + def test_path_rstrip(self): + scheme = 'https' + server = '13.37.73.31' + path = 'images/ubuntu-14.04.vmdk/' + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + query = urlparse.urlencode(params) + url = datastore.DatastoreURL(scheme, server, path, dc_path, ds_name) + expected_url = '%s://%s/folder/%s?%s' % ( + scheme, server, path.rstrip('/'), query) + self.assertEqual(expected_url, str(url)) + + def test_urlparse(self): + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + query = urlparse.urlencode(params) + url = 'https://13.37.73.31/folder/images/aa.vmdk?%s' % query + ds_url = datastore.DatastoreURL.urlparse(url) + self.assertEqual(url, str(ds_url)) + + def test_datastore_name(self): + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + query = urlparse.urlencode(params) + url = 'https://13.37.73.31/folder/images/aa.vmdk?%s' % query + ds_url = datastore.DatastoreURL.urlparse(url) + self.assertEqual(ds_name, ds_url.datastore_name) + + def test_datacenter_path(self): + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + query = urlparse.urlencode(params) + url = 'https://13.37.73.31/folder/images/aa.vmdk?%s' % query + ds_url = datastore.DatastoreURL.urlparse(url) + self.assertEqual(dc_path, ds_url.datacenter_path) + + def test_path(self): + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + path = 'images/aa.vmdk' + query = urlparse.urlencode(params) + url = 'https://13.37.73.31/folder/%s?%s' % (path, query) + ds_url = datastore.DatastoreURL.urlparse(url) + self.assertEqual(path, ds_url.path) + + @mock.patch('six.moves.http_client.HTTPSConnection') + def test_connect(self, mock_conn): + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + query = urlparse.urlencode(params) + url = 'https://13.37.73.31/folder/images/aa.vmdk?%s' % query + ds_url = datastore.DatastoreURL.urlparse(url) + cookie = mock.Mock() + ds_url.connect('PUT', 128, cookie) + mock_conn.assert_called_once_with('13.37.73.31') + + def test_get_transfer_ticket(self): + dc_path = 'datacenter-1' + ds_name = 'datastore-1' + params = {'dcPath': dc_path, 'dsName': ds_name} + query = urlparse.urlencode(params) + url = 'https://13.37.73.31/folder/images/aa.vmdk?%s' % query + session = mock.Mock() + session.invoke_api = mock.Mock() + + class Ticket(object): + id = 'fake_id' + session.invoke_api.return_value = Ticket() + ds_url = datastore.DatastoreURL.urlparse(url) + ticket = ds_url.get_transfer_ticket(session, 'PUT') + self.assertEqual('%s="%s"' % (constants.CGI_COOKIE_KEY, 'fake_id'), + ticket) diff --git a/oslo_vmware/tests/test_api.py b/oslo_vmware/tests/test_api.py new file mode 100644 index 00000000..0787254f --- /dev/null +++ b/oslo_vmware/tests/test_api.py @@ -0,0 +1,549 @@ +# coding=utf-8 +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit tests for session management and API invocation classes. +""" + +from eventlet import greenthread +import mock +import six +import suds + +from oslo_vmware import api +from oslo_vmware import exceptions +from oslo_vmware import pbm +from oslo_vmware.tests import base +from oslo_vmware import vim_util + + +class RetryDecoratorTest(base.TestCase): + """Tests for retry decorator class.""" + + def test_retry(self): + result = "RESULT" + + @api.RetryDecorator() + def func(*args, **kwargs): + return result + + self.assertEqual(result, func()) + + def func2(*args, **kwargs): + return result + + retry = api.RetryDecorator() + self.assertEqual(result, retry(func2)()) + self.assertTrue(retry._retry_count == 0) + + def test_retry_with_expected_exceptions(self): + result = "RESULT" + responses = [exceptions.VimSessionOverLoadException(None), + exceptions.VimSessionOverLoadException(None), + result] + + def func(*args, **kwargs): + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + sleep_time_incr = 0.01 + retry_count = 2 + retry = api.RetryDecorator(10, sleep_time_incr, 10, + (exceptions.VimSessionOverLoadException,)) + self.assertEqual(result, retry(func)()) + self.assertTrue(retry._retry_count == retry_count) + self.assertEqual(retry_count * sleep_time_incr, retry._sleep_time) + + def test_retry_with_max_retries(self): + responses = [exceptions.VimSessionOverLoadException(None), + exceptions.VimSessionOverLoadException(None), + exceptions.VimSessionOverLoadException(None)] + + def func(*args, **kwargs): + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + retry = api.RetryDecorator(2, 0, 0, + (exceptions.VimSessionOverLoadException,)) + self.assertRaises(exceptions.VimSessionOverLoadException, retry(func)) + self.assertTrue(retry._retry_count == 2) + + def test_retry_with_unexpected_exception(self): + + def func(*args, **kwargs): + raise exceptions.VimException(None) + + retry = api.RetryDecorator() + self.assertRaises(exceptions.VimException, retry(func)) + self.assertTrue(retry._retry_count == 0) + + +class VMwareAPISessionTest(base.TestCase): + """Tests for VMwareAPISession.""" + + SERVER_IP = '10.1.2.3' + PORT = 443 + USERNAME = 'admin' + PASSWORD = 'password' + + def setUp(self): + super(VMwareAPISessionTest, self).setUp() + patcher = mock.patch('oslo_vmware.vim.Vim') + self.addCleanup(patcher.stop) + self.VimMock = patcher.start() + self.VimMock.side_effect = lambda *args, **kw: mock.MagicMock() + self.cert_mock = mock.Mock() + + def _create_api_session(self, _create_session, retry_count=10, + task_poll_interval=1): + return api.VMwareAPISession(VMwareAPISessionTest.SERVER_IP, + VMwareAPISessionTest.USERNAME, + VMwareAPISessionTest.PASSWORD, + retry_count, + task_poll_interval, + 'https', + _create_session, + port=VMwareAPISessionTest.PORT, + cacert=self.cert_mock, + insecure=False) + + def test_vim(self): + api_session = self._create_api_session(False) + api_session.vim + self.VimMock.assert_called_with(protocol=api_session._scheme, + host=VMwareAPISessionTest.SERVER_IP, + port=VMwareAPISessionTest.PORT, + wsdl_url=api_session._vim_wsdl_loc, + cacert=self.cert_mock, + insecure=False) + + @mock.patch.object(pbm, 'Pbm') + def test_pbm(self, pbm_mock): + api_session = self._create_api_session(True) + vim_obj = api_session.vim + cookie = mock.Mock() + vim_obj.get_http_cookie.return_value = cookie + api_session._pbm_wsdl_loc = mock.Mock() + + pbm = mock.Mock() + pbm_mock.return_value = pbm + api_session._get_session_cookie = mock.Mock(return_value=cookie) + + self.assertEqual(pbm, api_session.pbm) + pbm.set_soap_cookie.assert_called_once_with(cookie) + + def test_create_session(self): + session = mock.Mock() + session.key = "12345" + api_session = self._create_api_session(False) + cookie = mock.Mock() + vim_obj = api_session.vim + vim_obj.Login.return_value = session + vim_obj.get_http_cookie.return_value = cookie + + pbm = mock.Mock() + api_session._pbm = pbm + + api_session._create_session() + session_manager = vim_obj.service_content.sessionManager + vim_obj.Login.assert_called_once_with( + session_manager, userName=VMwareAPISessionTest.USERNAME, + password=VMwareAPISessionTest.PASSWORD) + self.assertFalse(vim_obj.TerminateSession.called) + self.assertEqual(session.key, api_session._session_id) + pbm.set_soap_cookie.assert_called_once_with(cookie) + + def test_create_session_with_existing_session(self): + old_session_key = '12345' + new_session_key = '67890' + session = mock.Mock() + session.key = new_session_key + api_session = self._create_api_session(False) + api_session._session_id = old_session_key + vim_obj = api_session.vim + vim_obj.Login.return_value = session + + api_session._create_session() + session_manager = vim_obj.service_content.sessionManager + vim_obj.Login.assert_called_once_with( + session_manager, userName=VMwareAPISessionTest.USERNAME, + password=VMwareAPISessionTest.PASSWORD) + vim_obj.TerminateSession.assert_called_once_with( + session_manager, sessionId=[old_session_key]) + self.assertEqual(new_session_key, api_session._session_id) + + def test_invoke_api(self): + api_session = self._create_api_session(True) + response = mock.Mock() + + def api(*args, **kwargs): + return response + + module = mock.Mock() + module.api = api + ret = api_session.invoke_api(module, 'api') + self.assertEqual(response, ret) + + def test_logout_with_exception(self): + session = mock.Mock() + session.key = "12345" + api_session = self._create_api_session(False) + vim_obj = api_session.vim + vim_obj.Login.return_value = session + vim_obj.Logout.side_effect = exceptions.VimFaultException([], None) + api_session._create_session() + api_session.logout() + self.assertEqual("12345", api_session._session_id) + + def test_logout_no_session(self): + api_session = self._create_api_session(False) + vim_obj = api_session.vim + api_session.logout() + self.assertEqual(0, vim_obj.Logout.call_count) + + def test_logout_calls_vim_logout(self): + session = mock.Mock() + session.key = "12345" + api_session = self._create_api_session(False) + vim_obj = api_session.vim + vim_obj.Login.return_value = session + vim_obj.Logout.return_value = None + + api_session._create_session() + session_manager = vim_obj.service_content.sessionManager + vim_obj.Login.assert_called_once_with( + session_manager, userName=VMwareAPISessionTest.USERNAME, + password=VMwareAPISessionTest.PASSWORD) + api_session.logout() + vim_obj.Logout.assert_called_once_with( + session_manager) + self.assertIsNone(api_session._session_id) + + def test_invoke_api_with_expected_exception(self): + api_session = self._create_api_session(True) + api_session._create_session = mock.Mock() + vim_obj = api_session.vim + vim_obj.SessionIsActive.return_value = False + ret = mock.Mock() + responses = [exceptions.VimConnectionException(None), ret] + + def api(*args, **kwargs): + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + module = mock.Mock() + module.api = api + with mock.patch.object(greenthread, 'sleep'): + self.assertEqual(ret, api_session.invoke_api(module, 'api')) + api_session._create_session.assert_called_once_with() + + def test_invoke_api_not_recreate_session(self): + api_session = self._create_api_session(True) + api_session._create_session = mock.Mock() + vim_obj = api_session.vim + vim_obj.SessionIsActive.return_value = True + ret = mock.Mock() + responses = [exceptions.VimConnectionException(None), ret] + + def api(*args, **kwargs): + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + module = mock.Mock() + module.api = api + with mock.patch.object(greenthread, 'sleep'): + self.assertEqual(ret, api_session.invoke_api(module, 'api')) + self.assertFalse(api_session._create_session.called) + + def test_invoke_api_with_vim_fault_exception(self): + api_session = self._create_api_session(True) + + def api(*args, **kwargs): + raise exceptions.VimFaultException([], None) + + module = mock.Mock() + module.api = api + self.assertRaises(exceptions.VimFaultException, + api_session.invoke_api, + module, + 'api') + + def test_invoke_api_with_vim_fault_exception_details(self): + api_session = self._create_api_session(True) + fault_string = 'Invalid property.' + fault_list = [exceptions.INVALID_PROPERTY] + details = {u'name': suds.sax.text.Text(u'фира')} + + module = mock.Mock() + module.api.side_effect = exceptions.VimFaultException(fault_list, + fault_string, + details=details) + e = self.assertRaises(exceptions.InvalidPropertyException, + api_session.invoke_api, + module, + 'api') + details_str = u"{'name': 'фира'}" + expected_str = "%s\nFaults: %s\nDetails: %s" % (fault_string, + fault_list, + details_str) + self.assertEqual(expected_str, six.text_type(e)) + self.assertEqual(details, e.details) + + def test_invoke_api_with_empty_response(self): + api_session = self._create_api_session(True) + vim_obj = api_session.vim + vim_obj.SessionIsActive.return_value = True + + def api(*args, **kwargs): + raise exceptions.VimFaultException( + [exceptions.NOT_AUTHENTICATED], None) + + module = mock.Mock() + module.api = api + ret = api_session.invoke_api(module, 'api') + self.assertEqual([], ret) + vim_obj.SessionIsActive.assert_called_once_with( + vim_obj.service_content.sessionManager, + sessionID=api_session._session_id, + userName=api_session._session_username) + + def test_invoke_api_with_stale_session(self): + api_session = self._create_api_session(True) + api_session._create_session = mock.Mock() + vim_obj = api_session.vim + vim_obj.SessionIsActive.return_value = False + result = mock.Mock() + responses = [exceptions.VimFaultException( + [exceptions.NOT_AUTHENTICATED], None), result] + + def api(*args, **kwargs): + response = responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + module = mock.Mock() + module.api = api + with mock.patch.object(greenthread, 'sleep'): + ret = api_session.invoke_api(module, 'api') + self.assertEqual(result, ret) + vim_obj.SessionIsActive.assert_called_once_with( + vim_obj.service_content.sessionManager, + sessionID=api_session._session_id, + userName=api_session._session_username) + api_session._create_session.assert_called_once_with() + + def test_wait_for_task(self): + api_session = self._create_api_session(True) + task_info_list = [('queued', 0), ('running', 40), ('success', 100)] + task_info_list_size = len(task_info_list) + + def invoke_api_side_effect(module, method, *args, **kwargs): + (state, progress) = task_info_list.pop(0) + task_info = mock.Mock() + task_info.progress = progress + task_info.state = state + return task_info + + api_session.invoke_api = mock.Mock(side_effect=invoke_api_side_effect) + task = mock.Mock() + with mock.patch.object(greenthread, 'sleep'): + ret = api_session.wait_for_task(task) + self.assertEqual('success', ret.state) + self.assertEqual(100, ret.progress) + api_session.invoke_api.assert_called_with(vim_util, + 'get_object_property', + api_session.vim, task, + 'info') + self.assertEqual(task_info_list_size, + api_session.invoke_api.call_count) + + def test_wait_for_task_with_error_state(self): + api_session = self._create_api_session(True) + task_info_list = [('queued', 0), ('running', 40), ('error', -1)] + task_info_list_size = len(task_info_list) + + def invoke_api_side_effect(module, method, *args, **kwargs): + (state, progress) = task_info_list.pop(0) + task_info = mock.Mock() + task_info.progress = progress + task_info.state = state + return task_info + + api_session.invoke_api = mock.Mock(side_effect=invoke_api_side_effect) + task = mock.Mock() + with mock.patch.object(greenthread, 'sleep'): + self.assertRaises(exceptions.VMwareDriverException, + api_session.wait_for_task, + task) + api_session.invoke_api.assert_called_with(vim_util, + 'get_object_property', + api_session.vim, task, + 'info') + self.assertEqual(task_info_list_size, + api_session.invoke_api.call_count) + + def test_wait_for_task_with_invoke_api_exception(self): + api_session = self._create_api_session(True) + api_session.invoke_api = mock.Mock( + side_effect=exceptions.VimException(None)) + task = mock.Mock() + with mock.patch.object(greenthread, 'sleep'): + self.assertRaises(exceptions.VimException, + api_session.wait_for_task, + task) + api_session.invoke_api.assert_called_once_with(vim_util, + 'get_object_property', + api_session.vim, task, + 'info') + + def test_wait_for_lease_ready(self): + api_session = self._create_api_session(True) + lease_states = ['initializing', 'ready'] + num_states = len(lease_states) + + def invoke_api_side_effect(module, method, *args, **kwargs): + return lease_states.pop(0) + + api_session.invoke_api = mock.Mock(side_effect=invoke_api_side_effect) + lease = mock.Mock() + with mock.patch.object(greenthread, 'sleep'): + api_session.wait_for_lease_ready(lease) + api_session.invoke_api.assert_called_with(vim_util, + 'get_object_property', + api_session.vim, lease, + 'state') + self.assertEqual(num_states, api_session.invoke_api.call_count) + + def test_wait_for_lease_ready_with_error_state(self): + api_session = self._create_api_session(True) + responses = ['initializing', 'error', 'error_msg'] + + def invoke_api_side_effect(module, method, *args, **kwargs): + return responses.pop(0) + + api_session.invoke_api = mock.Mock(side_effect=invoke_api_side_effect) + lease = mock.Mock() + with mock.patch.object(greenthread, 'sleep'): + self.assertRaises(exceptions.VimException, + api_session.wait_for_lease_ready, + lease) + exp_calls = [mock.call(vim_util, 'get_object_property', + api_session.vim, lease, 'state')] * 2 + exp_calls.append(mock.call(vim_util, 'get_object_property', + api_session.vim, lease, 'error')) + self.assertEqual(exp_calls, api_session.invoke_api.call_args_list) + + def test_wait_for_lease_ready_with_unknown_state(self): + api_session = self._create_api_session(True) + + def invoke_api_side_effect(module, method, *args, **kwargs): + return 'unknown' + + api_session.invoke_api = mock.Mock(side_effect=invoke_api_side_effect) + lease = mock.Mock() + self.assertRaises(exceptions.VimException, + api_session.wait_for_lease_ready, + lease) + api_session.invoke_api.assert_called_once_with(vim_util, + 'get_object_property', + api_session.vim, + lease, 'state') + + def test_wait_for_lease_ready_with_invoke_api_exception(self): + api_session = self._create_api_session(True) + api_session.invoke_api = mock.Mock( + side_effect=exceptions.VimException(None)) + lease = mock.Mock() + self.assertRaises(exceptions.VimException, + api_session.wait_for_lease_ready, + lease) + api_session.invoke_api.assert_called_once_with( + vim_util, 'get_object_property', api_session.vim, lease, + 'state') + + def _poll_task_well_known_exceptions(self, fault, + expected_exception): + api_session = self._create_api_session(False) + + def fake_invoke_api(self, module, method, *args, **kwargs): + task_info = mock.Mock() + task_info.progress = -1 + task_info.state = 'error' + error = mock.Mock() + error.localizedMessage = "Error message" + error_fault = mock.Mock() + error_fault.__class__.__name__ = fault + error.fault = error_fault + task_info.error = error + return task_info + + with ( + mock.patch.object(api_session, 'invoke_api', fake_invoke_api) + ): + self.assertRaises(expected_exception, + api_session._poll_task, + 'fake-task') + + def test_poll_task_well_known_exceptions(self): + for k, v in six.iteritems(exceptions._fault_classes_registry): + self._poll_task_well_known_exceptions(k, v) + + def test_poll_task_unknown_exception(self): + _unknown_exceptions = { + 'NoDiskSpace': exceptions.VMwareDriverException, + 'RuntimeFault': exceptions.VMwareDriverException + } + + for k, v in six.iteritems(_unknown_exceptions): + self._poll_task_well_known_exceptions(k, v) + + def _create_subclass_exception(self): + class VimSubClass(exceptions.VMwareDriverException): + pass + return VimSubClass + + def test_register_fault_class(self): + exc = self._create_subclass_exception() + exceptions.register_fault_class('ValueError', exc) + self.assertEqual(exc, exceptions.get_fault_class('ValueError')) + + def test_register_fault_class_override(self): + exc = self._create_subclass_exception() + exceptions.register_fault_class(exceptions.ALREADY_EXISTS, exc) + self.assertEqual(exc, + exceptions.get_fault_class(exceptions.ALREADY_EXISTS)) + + def test_register_fault_classi_invalid(self): + self.assertRaises(TypeError, + exceptions.register_fault_class, + 'ValueError', ValueError) + + def test_update_pbm_wsdl_loc(self): + session = mock.Mock() + session.key = "12345" + api_session = self._create_api_session(False) + self.assertIsNone(api_session._pbm_wsdl_loc) + api_session.pbm_wsdl_loc_set('fake_wsdl') + self.assertEqual('fake_wsdl', api_session._pbm_wsdl_loc) diff --git a/oslo_vmware/tests/test_image_transfer.py b/oslo_vmware/tests/test_image_transfer.py new file mode 100644 index 00000000..4693226f --- /dev/null +++ b/oslo_vmware/tests/test_image_transfer.py @@ -0,0 +1,552 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit tests for functions and classes for image transfer. +""" + +import math + +from eventlet import greenthread +from eventlet import timeout +import mock + +from oslo_vmware import exceptions +from oslo_vmware import image_transfer +from oslo_vmware import rw_handles +from oslo_vmware.tests import base + + +class BlockingQueueTest(base.TestCase): + """Tests for BlockingQueue.""" + + def test_read(self): + max_size = 10 + chunk_size = 10 + max_transfer_size = 30 + queue = image_transfer.BlockingQueue(max_size, max_transfer_size) + + def get_side_effect(): + return [1] * chunk_size + + queue.get = mock.Mock(side_effect=get_side_effect) + while True: + data_item = queue.read(chunk_size) + if not data_item: + break + + self.assertEqual(max_transfer_size, queue._transferred) + exp_calls = [mock.call()] * int(math.ceil(float(max_transfer_size) / + chunk_size)) + self.assertEqual(exp_calls, queue.get.call_args_list) + + def test_write(self): + queue = image_transfer.BlockingQueue(10, 30) + queue.put = mock.Mock() + write_count = 10 + for _ in range(0, write_count): + queue.write([1]) + exp_calls = [mock.call([1])] * write_count + self.assertEqual(exp_calls, queue.put.call_args_list) + + def test_seek(self): + queue = image_transfer.BlockingQueue(10, 30) + self.assertRaises(IOError, queue.seek, 5) + + def test_tell(self): + queue = image_transfer.BlockingQueue(10, 30) + self.assertEqual(0, queue.tell()) + queue.get = mock.Mock(return_value=[1] * 10) + queue.read(10) + self.assertEqual(10, queue.tell()) + + +class ImageWriterTest(base.TestCase): + """Tests for ImageWriter class.""" + + def _create_image_writer(self): + self.image_service = mock.Mock() + self.context = mock.Mock() + self.input_file = mock.Mock() + self.image_id = mock.Mock() + return image_transfer.ImageWriter(self.context, self.input_file, + self.image_service, self.image_id) + + @mock.patch.object(greenthread, 'sleep') + def test_start(self, mock_sleep): + writer = self._create_image_writer() + status_list = ['queued', 'saving', 'active'] + + def image_service_show_side_effect(context, image_id): + status = status_list.pop(0) + return {'status': status} + + self.image_service.show.side_effect = image_service_show_side_effect + exp_calls = [mock.call(self.context, self.image_id)] * len(status_list) + writer.start() + self.assertTrue(writer.wait()) + self.image_service.update.assert_called_once_with(self.context, + self.image_id, {}, + data=self.input_file) + self.assertEqual(exp_calls, self.image_service.show.call_args_list) + + def test_start_with_killed_status(self): + writer = self._create_image_writer() + + def image_service_show_side_effect(_context, _image_id): + return {'status': 'killed'} + + self.image_service.show.side_effect = image_service_show_side_effect + writer.start() + self.assertRaises(exceptions.ImageTransferException, + writer.wait) + self.image_service.update.assert_called_once_with(self.context, + self.image_id, {}, + data=self.input_file) + self.image_service.show.assert_called_once_with(self.context, + self.image_id) + + def test_start_with_unknown_status(self): + writer = self._create_image_writer() + + def image_service_show_side_effect(_context, _image_id): + return {'status': 'unknown'} + + self.image_service.show.side_effect = image_service_show_side_effect + writer.start() + self.assertRaises(exceptions.ImageTransferException, + writer.wait) + self.image_service.update.assert_called_once_with(self.context, + self.image_id, {}, + data=self.input_file) + self.image_service.show.assert_called_once_with(self.context, + self.image_id) + + def test_start_with_image_service_show_exception(self): + writer = self._create_image_writer() + self.image_service.show.side_effect = RuntimeError() + writer.start() + self.assertRaises(exceptions.ImageTransferException, writer.wait) + self.image_service.update.assert_called_once_with(self.context, + self.image_id, {}, + data=self.input_file) + self.image_service.show.assert_called_once_with(self.context, + self.image_id) + + +class FileReadWriteTaskTest(base.TestCase): + """Tests for FileReadWriteTask class.""" + + def test_start(self): + data_items = [[1] * 10, [1] * 20, [1] * 5, []] + + def input_file_read_side_effect(arg): + self.assertEqual(arg, rw_handles.READ_CHUNKSIZE) + data = data_items[input_file_read_side_effect.i] + input_file_read_side_effect.i += 1 + return data + + input_file_read_side_effect.i = 0 + input_file = mock.Mock() + input_file.read.side_effect = input_file_read_side_effect + output_file = mock.Mock() + rw_task = image_transfer.FileReadWriteTask(input_file, output_file) + rw_task.start() + self.assertTrue(rw_task.wait()) + self.assertEqual(len(data_items), input_file.read.call_count) + + exp_calls = [] + for i in range(0, len(data_items)): + exp_calls.append(mock.call(data_items[i])) + self.assertEqual(exp_calls, output_file.write.call_args_list) + + self.assertEqual(len(data_items), + input_file.update_progress.call_count) + self.assertEqual(len(data_items), + output_file.update_progress.call_count) + + def test_start_with_read_exception(self): + input_file = mock.Mock() + input_file.read.side_effect = RuntimeError() + output_file = mock.Mock() + rw_task = image_transfer.FileReadWriteTask(input_file, output_file) + rw_task.start() + self.assertRaises(exceptions.ImageTransferException, rw_task.wait) + input_file.read.assert_called_once_with(rw_handles.READ_CHUNKSIZE) + + +class ImageTransferUtilityTest(base.TestCase): + """Tests for image_transfer utility methods.""" + + @mock.patch.object(timeout, 'Timeout') + @mock.patch.object(image_transfer, 'ImageWriter') + @mock.patch.object(image_transfer, 'FileReadWriteTask') + @mock.patch.object(image_transfer, 'BlockingQueue') + def test_start_transfer(self, fake_BlockingQueue, fake_FileReadWriteTask, + fake_ImageWriter, fake_Timeout): + + context = mock.Mock() + read_file_handle = mock.Mock() + read_file_handle.close = mock.Mock() + image_service = mock.Mock() + image_id = mock.Mock() + blocking_queue = mock.Mock() + + write_file_handle1 = mock.Mock() + write_file_handle1.close = mock.Mock() + write_file_handle2 = None + write_file_handles = [write_file_handle1, write_file_handle2] + + timeout_secs = 10 + blocking_queue_size = 10 + image_meta = {} + max_data_size = 30 + + fake_BlockingQueue.return_value = blocking_queue + fake_timer = mock.Mock() + fake_timer.cancel = mock.Mock() + fake_Timeout.return_value = fake_timer + + for write_file_handle in write_file_handles: + image_transfer._start_transfer(context, + timeout_secs, + read_file_handle, + max_data_size, + write_file_handle=write_file_handle, + image_service=image_service, + image_id=image_id, + image_meta=image_meta) + + exp_calls = [mock.call(blocking_queue_size, + max_data_size)] * len(write_file_handles) + self.assertEqual(exp_calls, + fake_BlockingQueue.call_args_list) + + exp_calls2 = [mock.call(read_file_handle, blocking_queue), + mock.call(blocking_queue, write_file_handle1), + mock.call(read_file_handle, blocking_queue)] + self.assertEqual(exp_calls2, + fake_FileReadWriteTask.call_args_list) + + exp_calls3 = mock.call(context, blocking_queue, image_service, + image_id, image_meta) + self.assertEqual(exp_calls3, + fake_ImageWriter.call_args) + + exp_calls4 = [mock.call(timeout_secs)] * len(write_file_handles) + self.assertEqual(exp_calls4, + fake_Timeout.call_args_list) + + self.assertEqual(len(write_file_handles), + fake_timer.cancel.call_count) + + self.assertEqual(len(write_file_handles), + read_file_handle.close.call_count) + + write_file_handle1.close.assert_called_once() + + @mock.patch.object(image_transfer, 'FileReadWriteTask') + @mock.patch.object(image_transfer, 'BlockingQueue') + def test_start_transfer_with_no_image_destination(self, fake_BlockingQueue, + fake_FileReadWriteTask): + + context = mock.Mock() + read_file_handle = mock.Mock() + write_file_handle = None + image_service = None + image_id = None + timeout_secs = 10 + image_meta = {} + blocking_queue_size = 10 + max_data_size = 30 + blocking_queue = mock.Mock() + + fake_BlockingQueue.return_value = blocking_queue + + self.assertRaises(ValueError, + image_transfer._start_transfer, + context, + timeout_secs, + read_file_handle, + max_data_size, + write_file_handle=write_file_handle, + image_service=image_service, + image_id=image_id, + image_meta=image_meta) + + fake_BlockingQueue.assert_called_once_with(blocking_queue_size, + max_data_size) + + fake_FileReadWriteTask.assert_called_once_with(read_file_handle, + blocking_queue) + + @mock.patch('oslo_vmware.rw_handles.FileWriteHandle') + @mock.patch('oslo_vmware.rw_handles.ImageReadHandle') + @mock.patch.object(image_transfer, '_start_transfer') + def test_download_flat_image( + self, + fake_transfer, + fake_rw_handles_ImageReadHandle, + fake_rw_handles_FileWriteHandle): + + context = mock.Mock() + image_id = mock.Mock() + image_service = mock.Mock() + image_service.download = mock.Mock() + image_service.download.return_value = 'fake_iter' + + fake_ImageReadHandle = 'fake_ImageReadHandle' + fake_FileWriteHandle = 'fake_FileWriteHandle' + cookies = [] + timeout_secs = 10 + image_size = 1000 + host = '127.0.0.1' + port = 443 + dc_path = 'dc1' + ds_name = 'ds1' + file_path = '/fake_path' + + fake_rw_handles_ImageReadHandle.return_value = fake_ImageReadHandle + fake_rw_handles_FileWriteHandle.return_value = fake_FileWriteHandle + + image_transfer.download_flat_image( + context, + timeout_secs, + image_service, + image_id, + image_size=image_size, + host=host, + port=port, + data_center_name=dc_path, + datastore_name=ds_name, + cookies=cookies, + file_path=file_path) + + image_service.download.assert_called_once_with(context, image_id) + + fake_rw_handles_ImageReadHandle.assert_called_once_with('fake_iter') + + fake_rw_handles_FileWriteHandle.assert_called_once_with( + host, + port, + dc_path, + ds_name, + cookies, + file_path, + image_size, + cacerts=None) + + fake_transfer.assert_called_once_with( + context, + timeout_secs, + fake_ImageReadHandle, + image_size, + write_file_handle=fake_FileWriteHandle) + + @mock.patch('oslo_vmware.rw_handles.VmdkWriteHandle') + @mock.patch.object(image_transfer, '_start_transfer') + def test_download_stream_optimized_data(self, fake_transfer, + fake_rw_handles_VmdkWriteHandle): + + context = mock.Mock() + session = mock.Mock() + read_handle = mock.Mock() + timeout_secs = 10 + image_size = 1000 + host = '127.0.0.1' + port = 443 + resource_pool = 'rp-1' + vm_folder = 'folder-1' + vm_import_spec = None + + fake_VmdkWriteHandle = mock.Mock() + fake_VmdkWriteHandle.get_imported_vm = mock.Mock() + fake_rw_handles_VmdkWriteHandle.return_value = fake_VmdkWriteHandle + + image_transfer.download_stream_optimized_data( + context, + timeout_secs, + read_handle, + session=session, + host=host, + port=port, + resource_pool=resource_pool, + vm_folder=vm_folder, + vm_import_spec=vm_import_spec, + image_size=image_size) + + fake_rw_handles_VmdkWriteHandle.assert_called_once_with( + session, + host, + port, + resource_pool, + vm_folder, + vm_import_spec, + image_size) + + fake_transfer.assert_called_once_with( + context, + timeout_secs, + read_handle, + image_size, + write_file_handle=fake_VmdkWriteHandle) + + fake_VmdkWriteHandle.get_imported_vm.assert_called_once() + + @mock.patch('oslo_vmware.rw_handles.ImageReadHandle') + @mock.patch.object(image_transfer, 'download_stream_optimized_data') + def test_download_stream_optimized_image( + self, fake_download_stream_optimized_data, + fake_rw_handles_ImageReadHandle): + + context = mock.Mock() + session = mock.Mock() + image_id = mock.Mock() + timeout_secs = 10 + image_size = 1000 + host = '127.0.0.1' + port = 443 + resource_pool = 'rp-1' + vm_folder = 'folder-1' + vm_import_spec = None + + fake_iter = 'fake_iter' + image_service = mock.Mock() + image_service.download = mock.Mock() + image_service.download.return_value = fake_iter + + fake_ImageReadHandle = 'fake_ImageReadHandle' + fake_rw_handles_ImageReadHandle.return_value = fake_ImageReadHandle + + image_transfer.download_stream_optimized_image( + context, + timeout_secs, + image_service, + image_id, + session=session, + host=host, + port=port, + resource_pool=resource_pool, + vm_folder=vm_folder, + vm_import_spec=vm_import_spec, + image_size=image_size) + + image_service.download.assert_called_once_with(context, image_id) + + fake_rw_handles_ImageReadHandle.assert_called_once_with(fake_iter) + + fake_download_stream_optimized_data.assert_called_once_with( + context, + timeout_secs, + fake_ImageReadHandle, + session=session, + host=host, + port=port, + resource_pool=resource_pool, + vm_folder=vm_folder, + vm_import_spec=vm_import_spec, + image_size=image_size) + + @mock.patch.object(image_transfer, '_start_transfer') + @mock.patch('oslo_vmware.rw_handles.VmdkReadHandle') + def test_copy_stream_optimized_disk( + self, vmdk_read_handle, start_transfer): + + read_handle = mock.sentinel.read_handle + vmdk_read_handle.return_value = read_handle + + context = mock.sentinel.context + timeout = mock.sentinel.timeout + write_handle = mock.Mock(name='/cinder/images/tmpAbcd.vmdk') + session = mock.sentinel.session + host = mock.sentinel.host + port = mock.sentinel.port + vm = mock.sentinel.vm + vmdk_file_path = mock.sentinel.vmdk_file_path + vmdk_size = mock.sentinel.vmdk_size + + image_transfer.copy_stream_optimized_disk( + context, timeout, write_handle, session=session, host=host, + port=port, vm=vm, vmdk_file_path=vmdk_file_path, + vmdk_size=vmdk_size) + + vmdk_read_handle.assert_called_once_with( + session, host, port, vm, vmdk_file_path, vmdk_size) + start_transfer.assert_called_once_with( + context, timeout, read_handle, vmdk_size, + write_file_handle=write_handle) + + @mock.patch('oslo_vmware.rw_handles.VmdkReadHandle') + @mock.patch.object(image_transfer, '_start_transfer') + def test_upload_image(self, fake_transfer, fake_rw_handles_VmdkReadHandle): + + context = mock.Mock() + image_id = mock.Mock() + owner_id = mock.Mock() + session = mock.Mock() + vm = mock.Mock() + image_service = mock.Mock() + + timeout_secs = 10 + image_size = 1000 + host = '127.0.0.1' + port = 443 + file_path = '/fake_path' + is_public = False + image_name = 'fake_image' + image_version = 1 + + fake_VmdkReadHandle = 'fake_VmdkReadHandle' + fake_rw_handles_VmdkReadHandle.return_value = fake_VmdkReadHandle + + image_transfer.upload_image(context, + timeout_secs, + image_service, + image_id, + owner_id, + session=session, + host=host, + port=port, + vm=vm, + vmdk_file_path=file_path, + vmdk_size=image_size, + is_public=is_public, + image_name=image_name, + image_version=image_version) + + fake_rw_handles_VmdkReadHandle.assert_called_once_with(session, + host, + port, + vm, + file_path, + image_size) + + image_metadata = {'disk_format': 'vmdk', + 'is_public': is_public, + 'name': image_name, + 'status': 'active', + 'container_format': 'bare', + 'size': 0, + 'properties': {'vmware_image_version': image_version, + 'vmware_disktype': 'streamOptimized', + 'owner_id': owner_id}} + + fake_transfer.assert_called_once_with(context, + timeout_secs, + fake_VmdkReadHandle, + 0, + image_service=image_service, + image_id=image_id, + image_meta=image_metadata) diff --git a/oslo_vmware/tests/test_pbm.py b/oslo_vmware/tests/test_pbm.py new file mode 100644 index 00000000..2894e5d6 --- /dev/null +++ b/oslo_vmware/tests/test_pbm.py @@ -0,0 +1,173 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit tests for PBM utility methods. +""" + +import os + +import mock +import six.moves.urllib.parse as urlparse +import six.moves.urllib.request as urllib + +from oslo_vmware import pbm +from oslo_vmware.tests import base + + +class PBMUtilityTest(base.TestCase): + """Tests for PBM utility methods.""" + + def test_get_all_profiles(self): + session = mock.Mock() + session.pbm = mock.Mock() + profile_ids = mock.Mock() + + def invoke_api_side_effect(module, method, *args, **kwargs): + self.assertEqual(session.pbm, module) + self.assertTrue(method in ['PbmQueryProfile', + 'PbmRetrieveContent']) + self.assertEqual(session.pbm.service_content.profileManager, + args[0]) + if method == 'PbmQueryProfile': + self.assertEqual('STORAGE', + kwargs['resourceType'].resourceType) + return profile_ids + self.assertEqual(profile_ids, kwargs['profileIds']) + + session.invoke_api.side_effect = invoke_api_side_effect + pbm.get_all_profiles(session) + self.assertEqual(2, session.invoke_api.call_count) + + def test_get_all_profiles_with_no_profiles(self): + session = mock.Mock() + session.pbm = mock.Mock() + session.invoke_api.return_value = [] + profiles = pbm.get_all_profiles(session) + session.invoke_api.assert_called_once_with( + session.pbm, + 'PbmQueryProfile', + session.pbm.service_content.profileManager, + resourceType=session.pbm.client.factory.create()) + self.assertEqual([], profiles) + + def _create_profile(self, profile_id, name): + profile = mock.Mock() + profile.profileId = profile_id + profile.name = name + return profile + + @mock.patch.object(pbm, 'get_all_profiles') + def test_get_profile_id_by_name(self, get_all_profiles): + profiles = [self._create_profile(str(i), 'profile-%d' % i) + for i in range(0, 10)] + get_all_profiles.return_value = profiles + + session = mock.Mock() + exp_profile_id = '5' + profile_id = pbm.get_profile_id_by_name(session, + 'profile-%s' % exp_profile_id) + self.assertEqual(exp_profile_id, profile_id) + get_all_profiles.assert_called_once_with(session) + + @mock.patch.object(pbm, 'get_all_profiles') + def test_get_profile_id_by_name_with_invalid_profile(self, + get_all_profiles): + profiles = [self._create_profile(str(i), 'profile-%d' % i) + for i in range(0, 10)] + get_all_profiles.return_value = profiles + + session = mock.Mock() + profile_id = pbm.get_profile_id_by_name(session, + ('profile-%s' % 11)) + self.assertFalse(profile_id) + get_all_profiles.assert_called_once_with(session) + + def test_filter_hubs_by_profile(self): + pbm_client = mock.Mock() + session = mock.Mock() + session.pbm = pbm_client + hubs = mock.Mock() + profile_id = 'profile-0' + + pbm.filter_hubs_by_profile(session, hubs, profile_id) + session.invoke_api.assert_called_once_with( + pbm_client, + 'PbmQueryMatchingHub', + pbm_client.service_content.placementSolver, + hubsToSearch=hubs, + profile=profile_id) + + def _create_datastore(self, value): + ds = mock.Mock() + ds.value = value + return ds + + def test_convert_datastores_to_hubs(self): + ds_values = [] + datastores = [] + for i in range(0, 10): + value = "ds-%d" % i + ds_values.append(value) + datastores.append(self._create_datastore(value)) + + pbm_client_factory = mock.Mock() + pbm_client_factory.create.side_effect = lambda *args: mock.Mock() + hubs = pbm.convert_datastores_to_hubs(pbm_client_factory, datastores) + self.assertEqual(len(datastores), len(hubs)) + hub_ids = [hub.hubId for hub in hubs] + self.assertEqual(set(ds_values), set(hub_ids)) + + def test_filter_datastores_by_hubs(self): + ds_values = [] + datastores = [] + for i in range(0, 10): + value = "ds-%d" % i + ds_values.append(value) + datastores.append(self._create_datastore(value)) + + hubs = [] + hub_ids = ds_values[0:int(len(ds_values) / 2)] + for hub_id in hub_ids: + hub = mock.Mock() + hub.hubId = hub_id + hubs.append(hub) + + filtered_ds = pbm.filter_datastores_by_hubs(hubs, datastores) + self.assertEqual(len(hubs), len(filtered_ds)) + filtered_ds_values = [ds.value for ds in filtered_ds] + self.assertEqual(set(hub_ids), set(filtered_ds_values)) + + def test_get_pbm_wsdl_location(self): + wsdl = pbm.get_pbm_wsdl_location(None) + self.assertIsNone(wsdl) + + def expected_wsdl(version): + driver_abs_dir = os.path.abspath(os.path.dirname(pbm.__file__)) + path = os.path.join(driver_abs_dir, 'wsdl', version, + 'pbmService.wsdl') + return urlparse.urljoin('file:', urllib.pathname2url(path)) + + with mock.patch('os.path.exists') as path_exists: + path_exists.return_value = True + wsdl = pbm.get_pbm_wsdl_location('5') + self.assertEqual(expected_wsdl('5'), wsdl) + wsdl = pbm.get_pbm_wsdl_location('5.5') + self.assertEqual(expected_wsdl('5.5'), wsdl) + wsdl = pbm.get_pbm_wsdl_location('5.5.1') + self.assertEqual(expected_wsdl('5.5'), wsdl) + path_exists.return_value = False + wsdl = pbm.get_pbm_wsdl_location('5.5') + self.assertIsNone(wsdl) diff --git a/oslo_vmware/tests/test_rw_handles.py b/oslo_vmware/tests/test_rw_handles.py new file mode 100644 index 00000000..f6c5623f --- /dev/null +++ b/oslo_vmware/tests/test_rw_handles.py @@ -0,0 +1,302 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit tests for read and write handles for image transfer. +""" + +import mock +import six + +from oslo_vmware import exceptions +from oslo_vmware import rw_handles +from oslo_vmware.tests import base +from oslo_vmware import vim_util + + +class FileHandleTest(base.TestCase): + """Tests for FileHandle.""" + + def test_close(self): + file_handle = mock.Mock() + vmw_http_file = rw_handles.FileHandle(file_handle) + vmw_http_file.close() + file_handle.close.assert_called_once_with() + + def test_find_vmdk_url(self): + device_url_0 = mock.Mock() + device_url_0.disk = False + device_url_1 = mock.Mock() + device_url_1.disk = True + device_url_1.url = 'https://*/ds1/vm1.vmdk' + lease_info = mock.Mock() + lease_info.deviceUrl = [device_url_0, device_url_1] + host = '10.1.2.3' + port = 443 + exp_url = 'https://%s:%d/ds1/vm1.vmdk' % (host, port) + vmw_http_file = rw_handles.FileHandle(None) + self.assertEqual(exp_url, vmw_http_file._find_vmdk_url(lease_info, + host, + port)) + + +class FileWriteHandleTest(base.TestCase): + """Tests for FileWriteHandle.""" + + def setUp(self): + super(FileWriteHandleTest, self).setUp() + + vim_cookie = mock.Mock() + vim_cookie.name = 'name' + vim_cookie.value = 'value' + + self._conn = mock.Mock() + patcher = mock.patch( + 'urllib3.connection.HTTPConnection') + self.addCleanup(patcher.stop) + HTTPConnectionMock = patcher.start() + HTTPConnectionMock.return_value = self._conn + + self.vmw_http_write_file = rw_handles.FileWriteHandle( + '10.1.2.3', 443, 'dc-0', 'ds-0', [vim_cookie], '1.vmdk', 100, + 'http') + + def test_write(self): + self.vmw_http_write_file.write(None) + self._conn.send.assert_called_once_with(None) + + def test_close(self): + self.vmw_http_write_file.close() + self._conn.getresponse.assert_called_once_with() + self._conn.close.assert_called_once_with() + + +class VmdkWriteHandleTest(base.TestCase): + """Tests for VmdkWriteHandle.""" + + def setUp(self): + super(VmdkWriteHandleTest, self).setUp() + self._conn = mock.Mock() + patcher = mock.patch( + 'urllib3.connection.HTTPConnection') + self.addCleanup(patcher.stop) + HTTPConnectionMock = patcher.start() + HTTPConnectionMock.return_value = self._conn + + def _create_mock_session(self, disk=True, progress=-1): + device_url = mock.Mock() + device_url.disk = disk + device_url.url = 'http://*/ds/disk1.vmdk' + lease_info = mock.Mock() + lease_info.deviceUrl = [device_url] + session = mock.Mock() + + def session_invoke_api_side_effect(module, method, *args, **kwargs): + if module == session.vim: + if method == 'ImportVApp': + return mock.Mock() + elif method == 'HttpNfcLeaseProgress': + self.assertEqual(progress, kwargs['percent']) + return + return lease_info + + session.invoke_api.side_effect = session_invoke_api_side_effect + vim_cookie = mock.Mock() + vim_cookie.name = 'name' + vim_cookie.value = 'value' + session.vim.client.options.transport.cookiejar = [vim_cookie] + return session + + def test_init_failure(self): + session = self._create_mock_session(False) + self.assertRaises(exceptions.VimException, + rw_handles.VmdkWriteHandle, + session, + '10.1.2.3', + 443, + 'rp-1', + 'folder-1', + None, + 100) + + def test_write(self): + session = self._create_mock_session() + handle = rw_handles.VmdkWriteHandle(session, '10.1.2.3', 443, + 'rp-1', 'folder-1', None, + 100) + data = [1] * 10 + handle.write(data) + self.assertEqual(len(data), handle._bytes_written) + self._conn.send.assert_called_once_with(data) + + def test_update_progress(self): + vmdk_size = 100 + data_size = 10 + session = self._create_mock_session(True, 10) + handle = rw_handles.VmdkWriteHandle(session, '10.1.2.3', 443, + 'rp-1', 'folder-1', None, + vmdk_size) + handle.write([1] * data_size) + handle.update_progress() + + def test_update_progress_with_error(self): + session = self._create_mock_session(True, 10) + handle = rw_handles.VmdkWriteHandle(session, '10.1.2.3', 443, + 'rp-1', 'folder-1', None, + 100) + session.invoke_api.side_effect = exceptions.VimException(None) + self.assertRaises(exceptions.VimException, handle.update_progress) + + def test_close(self): + session = self._create_mock_session() + handle = rw_handles.VmdkWriteHandle(session, '10.1.2.3', 443, + 'rp-1', 'folder-1', None, + 100) + + def session_invoke_api_side_effect(module, method, *args, **kwargs): + if module == vim_util and method == 'get_object_property': + return 'ready' + self.assertEqual(session.vim, module) + self.assertEqual('HttpNfcLeaseComplete', method) + + session.invoke_api = mock.Mock( + side_effect=session_invoke_api_side_effect) + handle.close() + self.assertEqual(2, session.invoke_api.call_count) + + +class VmdkReadHandleTest(base.TestCase): + """Tests for VmdkReadHandle.""" + + def setUp(self): + super(VmdkReadHandleTest, self).setUp() + + send_patcher = mock.patch('requests.sessions.Session.send') + self.addCleanup(send_patcher.stop) + send_mock = send_patcher.start() + self._response = mock.Mock() + send_mock.return_value = self._response + + def _create_mock_session(self, disk=True, progress=-1): + device_url = mock.Mock() + device_url.disk = disk + device_url.url = 'http://*/ds/disk1.vmdk' + lease_info = mock.Mock() + lease_info.deviceUrl = [device_url] + session = mock.Mock() + + def session_invoke_api_side_effect(module, method, *args, **kwargs): + if module == session.vim: + if method == 'ExportVm': + return mock.Mock() + elif method == 'HttpNfcLeaseProgress': + self.assertEqual(progress, kwargs['percent']) + return + return lease_info + + session.invoke_api.side_effect = session_invoke_api_side_effect + vim_cookie = mock.Mock() + vim_cookie.name = 'name' + vim_cookie.value = 'value' + session.vim.client.options.transport.cookiejar = [vim_cookie] + return session + + def test_init_failure(self): + session = self._create_mock_session(False) + self.assertRaises(exceptions.VimException, + rw_handles.VmdkReadHandle, + session, + '10.1.2.3', + 443, + 'vm-1', + '[ds] disk1.vmdk', + 100) + + def test_read(self): + chunk_size = rw_handles.READ_CHUNKSIZE + session = self._create_mock_session() + self._response.raw.read.return_value = [1] * chunk_size + handle = rw_handles.VmdkReadHandle(session, '10.1.2.3', 443, + 'vm-1', '[ds] disk1.vmdk', + chunk_size * 10) + handle.read(chunk_size) + self.assertEqual(chunk_size, handle._bytes_read) + self._response.raw.read.assert_called_once_with(chunk_size) + + def test_update_progress(self): + chunk_size = rw_handles.READ_CHUNKSIZE + vmdk_size = chunk_size * 10 + session = self._create_mock_session(True, 10) + self._response.raw.read.return_value = [1] * chunk_size + handle = rw_handles.VmdkReadHandle(session, '10.1.2.3', 443, + 'vm-1', '[ds] disk1.vmdk', + vmdk_size) + handle.read(chunk_size) + handle.update_progress() + self._response.raw.read.assert_called_once_with(chunk_size) + + def test_update_progress_with_error(self): + session = self._create_mock_session(True, 10) + handle = rw_handles.VmdkReadHandle(session, '10.1.2.3', 443, + 'vm-1', '[ds] disk1.vmdk', + 100) + session.invoke_api.side_effect = exceptions.VimException(None) + self.assertRaises(exceptions.VimException, handle.update_progress) + + def test_close(self): + session = self._create_mock_session() + handle = rw_handles.VmdkReadHandle(session, '10.1.2.3', 443, + 'vm-1', '[ds] disk1.vmdk', + 100) + + def session_invoke_api_side_effect(module, method, *args, **kwargs): + if module == vim_util and method == 'get_object_property': + return 'ready' + self.assertEqual(session.vim, module) + self.assertEqual('HttpNfcLeaseComplete', method) + + session.invoke_api = mock.Mock( + side_effect=session_invoke_api_side_effect) + handle.close() + self.assertEqual(2, session.invoke_api.call_count) + + +class ImageReadHandleTest(base.TestCase): + """Tests for ImageReadHandle.""" + + def test_read(self): + max_items = 10 + item = [1] * 10 + + class ImageReadIterator(six.Iterator): + + def __init__(self): + self.num_items = 0 + + def __iter__(self): + return self + + def __next__(self): + if (self.num_items < max_items): + self.num_items += 1 + return item + raise StopIteration + + next = __next__ + + handle = rw_handles.ImageReadHandle(ImageReadIterator()) + for _ in range(0, max_items): + self.assertEqual(item, handle.read(10)) + self.assertFalse(handle.read(10)) diff --git a/oslo_vmware/tests/test_service.py b/oslo_vmware/tests/test_service.py new file mode 100644 index 00000000..52aedc25 --- /dev/null +++ b/oslo_vmware/tests/test_service.py @@ -0,0 +1,446 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import requests +import six +import six.moves.http_client as httplib +import suds + +from oslo_vmware import exceptions +from oslo_vmware import service +from oslo_vmware.tests import base +from oslo_vmware import vim_util + + +class ServiceMessagePluginTest(base.TestCase): + """Test class for ServiceMessagePlugin.""" + + def test_add_attribute_for_value(self): + node = mock.Mock() + node.name = 'value' + plugin = service.ServiceMessagePlugin() + plugin.add_attribute_for_value(node) + node.set.assert_called_once_with('xsi:type', 'xsd:string') + + def test_marshalled(self): + plugin = service.ServiceMessagePlugin() + context = mock.Mock() + plugin.marshalled(context) + context.envelope.prune.assert_called_once_with() + context.envelope.walk.assert_called_once_with( + plugin.add_attribute_for_value) + + +class ServiceTest(base.TestCase): + + def setUp(self): + super(ServiceTest, self).setUp() + patcher = mock.patch('suds.client.Client') + self.addCleanup(patcher.stop) + self.SudsClientMock = patcher.start() + + def test_retrieve_properties_ex_fault_checker_with_empty_response(self): + try: + service.Service._retrieve_properties_ex_fault_checker(None) + assert False + except exceptions.VimFaultException as ex: + self.assertEqual([exceptions.NOT_AUTHENTICATED], + ex.fault_list) + + def test_retrieve_properties_ex_fault_checker(self): + fault_list = ['FileFault', 'VimFault'] + missing_set = [] + for fault in fault_list: + missing_elem = mock.Mock() + missing_elem.fault.fault.__class__.__name__ = fault + missing_set.append(missing_elem) + obj_cont = mock.Mock() + obj_cont.missingSet = missing_set + response = mock.Mock() + response.objects = [obj_cont] + + try: + service.Service._retrieve_properties_ex_fault_checker(response) + assert False + except exceptions.VimFaultException as ex: + self.assertEqual(fault_list, ex.fault_list) + + def test_request_handler(self): + managed_object = 'VirtualMachine' + resp = mock.Mock() + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + return resp + + svc_obj = service.Service() + attr_name = 'powerOn' + service_mock = svc_obj.client.service + setattr(service_mock, attr_name, side_effect) + ret = svc_obj.powerOn(managed_object) + self.assertEqual(resp, ret) + + def test_request_handler_with_retrieve_properties_ex_fault(self): + managed_object = 'Datacenter' + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + return None + + svc_obj = service.Service() + attr_name = 'retrievePropertiesEx' + service_mock = svc_obj.client.service + setattr(service_mock, attr_name, side_effect) + self.assertRaises(exceptions.VimFaultException, + svc_obj.retrievePropertiesEx, + managed_object) + + def test_request_handler_with_web_fault(self): + managed_object = 'VirtualMachine' + fault_list = ['Fault'] + + doc = mock.Mock() + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + fault = mock.Mock(faultstring="MyFault") + + fault_children = mock.Mock() + fault_children.name = "name" + fault_children.getText.return_value = "value" + child = mock.Mock() + child.get.return_value = fault_list[0] + child.getChildren.return_value = [fault_children] + detail = mock.Mock() + detail.getChildren.return_value = [child] + doc.childAtPath.return_value = detail + raise suds.WebFault(fault, doc) + + svc_obj = service.Service() + service_mock = svc_obj.client.service + setattr(service_mock, 'powerOn', side_effect) + + ex = self.assertRaises(exceptions.VimFaultException, svc_obj.powerOn, + managed_object) + + self.assertEqual(fault_list, ex.fault_list) + self.assertEqual({'name': 'value'}, ex.details) + self.assertEqual("MyFault", ex.msg) + doc.childAtPath.assertCalledOnceWith('/detail') + + def test_request_handler_with_empty_web_fault_doc(self): + + def side_effect(mo, **kwargs): + fault = mock.Mock(faultstring="MyFault") + raise suds.WebFault(fault, None) + + svc_obj = service.Service() + service_mock = svc_obj.client.service + setattr(service_mock, 'powerOn', side_effect) + + ex = self.assertRaises(exceptions.VimFaultException, + svc_obj.powerOn, + 'VirtualMachine') + self.assertEqual([], ex.fault_list) + self.assertEqual({}, ex.details) + self.assertEqual("MyFault", ex.msg) + + def test_request_handler_with_vc51_web_fault(self): + managed_object = 'VirtualMachine' + fault_list = ['Fault'] + + doc = mock.Mock() + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + fault = mock.Mock(faultstring="MyFault") + + fault_children = mock.Mock() + fault_children.name = "name" + fault_children.getText.return_value = "value" + child = mock.Mock() + child.get.return_value = fault_list[0] + child.getChildren.return_value = [fault_children] + detail = mock.Mock() + detail.getChildren.return_value = [child] + doc.childAtPath.side_effect = [None, detail] + raise suds.WebFault(fault, doc) + + svc_obj = service.Service() + service_mock = svc_obj.client.service + setattr(service_mock, 'powerOn', side_effect) + + ex = self.assertRaises(exceptions.VimFaultException, svc_obj.powerOn, + managed_object) + + self.assertEqual(fault_list, ex.fault_list) + self.assertEqual({'name': 'value'}, ex.details) + self.assertEqual("MyFault", ex.msg) + exp_calls = [mock.call('/detail'), + mock.call('/Envelope/Body/Fault/detail')] + self.assertEqual(exp_calls, doc.childAtPath.call_args_list) + + def test_request_handler_with_attribute_error(self): + managed_object = 'VirtualMachine' + svc_obj = service.Service() + # no powerOn method in Service + service_mock = mock.Mock(spec=service.Service) + svc_obj.client.service = service_mock + self.assertRaises(exceptions.VimAttributeException, + svc_obj.powerOn, + managed_object) + + def test_request_handler_with_http_cannot_send_error(self): + managed_object = 'VirtualMachine' + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + raise httplib.CannotSendRequest() + + svc_obj = service.Service() + attr_name = 'powerOn' + service_mock = svc_obj.client.service + setattr(service_mock, attr_name, side_effect) + self.assertRaises(exceptions.VimSessionOverLoadException, + svc_obj.powerOn, + managed_object) + + def test_request_handler_with_http_response_not_ready_error(self): + managed_object = 'VirtualMachine' + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + raise httplib.ResponseNotReady() + + svc_obj = service.Service() + attr_name = 'powerOn' + service_mock = svc_obj.client.service + setattr(service_mock, attr_name, side_effect) + self.assertRaises(exceptions.VimSessionOverLoadException, + svc_obj.powerOn, + managed_object) + + def test_request_handler_with_http_cannot_send_header_error(self): + managed_object = 'VirtualMachine' + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + raise httplib.CannotSendHeader() + + svc_obj = service.Service() + attr_name = 'powerOn' + service_mock = svc_obj.client.service + setattr(service_mock, attr_name, side_effect) + self.assertRaises(exceptions.VimSessionOverLoadException, + svc_obj.powerOn, + managed_object) + + def test_request_handler_with_connection_error(self): + managed_object = 'VirtualMachine' + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + raise requests.ConnectionError() + + svc_obj = service.Service() + attr_name = 'powerOn' + service_mock = svc_obj.client.service + setattr(service_mock, attr_name, side_effect) + self.assertRaises(exceptions.VimConnectionException, + svc_obj.powerOn, + managed_object) + + def test_request_handler_with_http_error(self): + managed_object = 'VirtualMachine' + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + raise requests.HTTPError() + + svc_obj = service.Service() + attr_name = 'powerOn' + service_mock = svc_obj.client.service + setattr(service_mock, attr_name, side_effect) + self.assertRaises(exceptions.VimConnectionException, + svc_obj.powerOn, + managed_object) + + @mock.patch.object(vim_util, 'get_moref', return_value=None) + def test_request_handler_no_value(self, mock_moref): + managed_object = 'VirtualMachine' + svc_obj = service.Service() + ret = svc_obj.UnregisterVM(managed_object) + self.assertIsNone(ret) + + def _test_request_handler_with_exception(self, message, exception): + managed_object = 'VirtualMachine' + + def side_effect(mo, **kwargs): + self.assertEqual(managed_object, mo._type) + self.assertEqual(managed_object, mo.value) + raise Exception(message) + + svc_obj = service.Service() + attr_name = 'powerOn' + service_mock = svc_obj.client.service + setattr(service_mock, attr_name, side_effect) + self.assertRaises(exception, svc_obj.powerOn, managed_object) + + def test_request_handler_with_address_in_use_error(self): + self._test_request_handler_with_exception( + service.ADDRESS_IN_USE_ERROR, + exceptions.VimSessionOverLoadException) + + def test_request_handler_with_conn_abort_error(self): + self._test_request_handler_with_exception( + service.CONN_ABORT_ERROR, exceptions.VimSessionOverLoadException) + + def test_request_handler_with_resp_not_xml_error(self): + self._test_request_handler_with_exception( + service.RESP_NOT_XML_ERROR, exceptions.VimSessionOverLoadException) + + def test_request_handler_with_generic_error(self): + self._test_request_handler_with_exception( + 'GENERIC_ERROR', exceptions.VimException) + + def test_get_session_cookie(self): + svc_obj = service.Service() + cookie_value = 'xyz' + cookie = mock.Mock() + cookie.name = 'vmware_soap_session' + cookie.value = cookie_value + svc_obj.client.options.transport.cookiejar = [cookie] + self.assertEqual(cookie_value, svc_obj.get_http_cookie()) + + def test_get_session_cookie_with_no_cookie(self): + svc_obj = service.Service() + cookie = mock.Mock() + cookie.name = 'cookie' + cookie.value = 'xyz' + svc_obj.client.options.transport.cookiejar = [cookie] + self.assertIsNone(svc_obj.get_http_cookie()) + + +class MemoryCacheTest(base.TestCase): + """Test class for MemoryCache.""" + + def test_get_set(self): + cache = service.MemoryCache() + cache.put('key1', 'value1') + cache.put('key2', 'value2') + self.assertEqual('value1', cache.get('key1')) + self.assertEqual('value2', cache.get('key2')) + self.assertEqual(None, cache.get('key3')) + + @mock.patch('suds.reader.DocumentReader.download') + def test_shared_cache(self, mock_reader): + cache1 = service.Service().client.options.cache + cache2 = service.Service().client.options.cache + self.assertIs(cache1, cache2) + + @mock.patch('oslo.utils.timeutils.utcnow_ts') + def test_cache_timeout(self, mock_utcnow_ts): + mock_utcnow_ts.side_effect = [100, 125, 150, 175, 195, 200, 225] + + cache = service.MemoryCache() + cache.put('key1', 'value1', 10) + cache.put('key2', 'value2', 75) + cache.put('key3', 'value3', 100) + + self.assertIsNone(cache.get('key1')) + self.assertEqual('value2', cache.get('key2')) + self.assertIsNone(cache.get('key2')) + self.assertEqual('value3', cache.get('key3')) + + +class RequestsTransportTest(base.TestCase): + """Tests for RequestsTransport.""" + + def test_open(self): + transport = service.RequestsTransport() + + data = "Hello World" + resp = mock.Mock(content=data) + transport.session.get = mock.Mock(return_value=resp) + + request = mock.Mock(url=mock.sentinel.url) + self.assertEqual(data, + transport.open(request).getvalue()) + transport.session.get.assert_called_once_with(mock.sentinel.url, + verify=transport.verify) + + def test_send(self): + transport = service.RequestsTransport() + + resp = mock.Mock(status_code=mock.sentinel.status_code, + headers=mock.sentinel.headers, + content=mock.sentinel.content) + transport.session.post = mock.Mock(return_value=resp) + + request = mock.Mock(url=mock.sentinel.url, + message=mock.sentinel.message, + headers=mock.sentinel.req_headers) + reply = transport.send(request) + + self.assertEqual(mock.sentinel.status_code, reply.code) + self.assertEqual(mock.sentinel.headers, reply.headers) + self.assertEqual(mock.sentinel.content, reply.message) + + @mock.patch('os.path.getsize') + def test_send_with_local_file_url(self, get_size_mock): + transport = service.RequestsTransport() + + url = 'file:///foo' + request = requests.PreparedRequest() + request.url = url + + data = b"Hello World" + get_size_mock.return_value = len(data) + + def readinto_mock(buf): + buf[0:] = data + + if six.PY3: + builtin_open = 'builtins.open' + open_mock = mock.MagicMock(name='file_handle', + spec=open) + import _io + file_spec = list(set(dir(_io.TextIOWrapper)).union( + set(dir(_io.BytesIO)))) + else: + builtin_open = '__builtin__.open' + open_mock = mock.MagicMock(name='file_handle', + spec=file) + file_spec = file + + file_handle = mock.MagicMock(spec=file_spec) + file_handle.write.return_value = None + file_handle.__enter__.return_value = file_handle + file_handle.readinto.side_effect = readinto_mock + open_mock.return_value = file_handle + + with mock.patch(builtin_open, open_mock, create=True): + resp = transport.session.send(request) + self.assertEqual(data, resp.content) diff --git a/oslo_vmware/tests/test_vim.py b/oslo_vmware/tests/test_vim.py new file mode 100644 index 00000000..94c71a82 --- /dev/null +++ b/oslo_vmware/tests/test_vim.py @@ -0,0 +1,110 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit tests for classes to invoke VMware VI SOAP calls. +""" + +import mock +from oslo_i18n import fixture as i18n_fixture + +from oslo_vmware._i18n import _ +from oslo_vmware import exceptions +from oslo_vmware.tests import base +from oslo_vmware import vim + + +class VimTest(base.TestCase): + """Test class for Vim.""" + + def setUp(self): + super(VimTest, self).setUp() + patcher = mock.patch('suds.client.Client') + self.addCleanup(patcher.stop) + self.SudsClientMock = patcher.start() + self.useFixture(i18n_fixture.ToggleLazy(True)) + + @mock.patch.object(vim.Vim, '__getattr__', autospec=True) + def test_service_content(self, getattr_mock): + getattr_ret = mock.Mock() + getattr_mock.side_effect = lambda *args: getattr_ret + vim_obj = vim.Vim() + vim_obj.service_content + getattr_mock.assert_called_once_with(vim_obj, 'RetrieveServiceContent') + getattr_ret.assert_called_once_with('ServiceInstance') + self.assertEqual(self.SudsClientMock.return_value, vim_obj.client) + self.assertEqual(getattr_ret.return_value, vim_obj.service_content) + + def test_exception_summary_exception_as_list(self): + # assert that if a list is fed to the VimException object + # that it will error. + self.assertRaises(ValueError, + exceptions.VimException, + [], ValueError('foo')) + + def test_exception_summary_string(self): + e = exceptions.VimException(_("string"), ValueError("foo")) + string = str(e) + self.assertEqual("string\nCause: foo", string) + + def test_vim_fault_exception_string(self): + self.assertRaises(ValueError, + exceptions.VimFaultException, + "bad", ValueError("argument")) + + def test_vim_fault_exception(self): + vfe = exceptions.VimFaultException([ValueError("example")], _("cause")) + string = str(vfe) + self.assertEqual("cause\nFaults: [ValueError('example',)]", string) + + def test_vim_fault_exception_with_cause_and_details(self): + vfe = exceptions.VimFaultException([ValueError("example")], + "MyMessage", + "FooBar", + {'foo': 'bar'}) + string = str(vfe) + self.assertEqual("MyMessage\n" + "Cause: FooBar\n" + "Faults: [ValueError('example',)]\n" + "Details: {'foo': 'bar'}", + string) + + def test_configure_non_default_host_port(self): + vim_obj = vim.Vim('https', 'www.test.com', 12345) + self.assertEqual('https://www.test.com:12345/sdk/vimService.wsdl', + vim_obj.wsdl_url) + self.assertEqual('https://www.test.com:12345/sdk', + vim_obj.soap_url) + + def test_configure_ipv6(self): + vim_obj = vim.Vim('https', '::1') + self.assertEqual('https://[::1]/sdk/vimService.wsdl', + vim_obj.wsdl_url) + self.assertEqual('https://[::1]/sdk', + vim_obj.soap_url) + + def test_configure_ipv6_and_non_default_host_port(self): + vim_obj = vim.Vim('https', '::1', 12345) + self.assertEqual('https://[::1]:12345/sdk/vimService.wsdl', + vim_obj.wsdl_url) + self.assertEqual('https://[::1]:12345/sdk', + vim_obj.soap_url) + + def test_configure_with_wsdl_url_override(self): + vim_obj = vim.Vim('https', 'www.example.com', + wsdl_url='https://test.com/sdk/vimService.wsdl') + self.assertEqual('https://test.com/sdk/vimService.wsdl', + vim_obj.wsdl_url) + self.assertEqual('https://www.example.com/sdk', vim_obj.soap_url) diff --git a/oslo_vmware/tests/test_vim_util.py b/oslo_vmware/tests/test_vim_util.py new file mode 100644 index 00000000..a306d009 --- /dev/null +++ b/oslo_vmware/tests/test_vim_util.py @@ -0,0 +1,363 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit tests for VMware API utility module. +""" + +import collections + +import mock + +from oslo_vmware.tests import base +from oslo_vmware import vim_util + + +class VimUtilTest(base.TestCase): + """Test class for utility methods in vim_util.""" + + def test_get_moref(self): + moref = vim_util.get_moref("vm-0", "VirtualMachine") + self.assertEqual("vm-0", moref.value) + self.assertEqual("VirtualMachine", moref._type) + + def test_build_selection_spec(self): + client_factory = mock.Mock() + sel_spec = vim_util.build_selection_spec(client_factory, "test") + self.assertEqual("test", sel_spec.name) + + def test_build_traversal_spec(self): + client_factory = mock.Mock() + sel_spec = mock.Mock() + traversal_spec = vim_util.build_traversal_spec(client_factory, + 'dc_to_hf', + 'Datacenter', + 'hostFolder', False, + [sel_spec]) + self.assertEqual("dc_to_hf", traversal_spec.name) + self.assertEqual("hostFolder", traversal_spec.path) + self.assertEqual([sel_spec], traversal_spec.selectSet) + self.assertFalse(traversal_spec.skip) + self.assertEqual("Datacenter", traversal_spec.type) + + @mock.patch.object(vim_util, 'build_selection_spec') + def test_build_recursive_traversal_spec(self, build_selection_spec_mock): + sel_spec = mock.Mock() + rp_to_rp_sel_spec = mock.Mock() + rp_to_vm_sel_spec = mock.Mock() + + def build_sel_spec_side_effect(client_factory, name): + if name == 'visitFolders': + return sel_spec + elif name == 'rp_to_rp': + return rp_to_rp_sel_spec + elif name == 'rp_to_vm': + return rp_to_vm_sel_spec + else: + return None + + build_selection_spec_mock.side_effect = build_sel_spec_side_effect + traversal_spec_dict = {'dc_to_hf': {'type': 'Datacenter', + 'path': 'hostFolder', + 'skip': False, + 'selectSet': [sel_spec]}, + 'dc_to_vmf': {'type': 'Datacenter', + 'path': 'vmFolder', + 'skip': False, + 'selectSet': [sel_spec]}, + 'dc_to_netf': {'type': 'Datacenter', + 'path': 'networkFolder', + 'skip': False, + 'selectSet': [sel_spec]}, + 'h_to_vm': {'type': 'HostSystem', + 'path': 'vm', + 'skip': False, + 'selectSet': [sel_spec]}, + 'cr_to_h': {'type': 'ComputeResource', + 'path': 'host', + 'skip': False, + 'selectSet': []}, + 'cr_to_ds': {'type': 'ComputeResource', + 'path': 'datastore', + 'skip': False, + 'selectSet': []}, + 'cr_to_rp': {'type': 'ComputeResource', + 'path': 'resourcePool', + 'skip': False, + 'selectSet': [rp_to_rp_sel_spec, + rp_to_vm_sel_spec]}, + 'cr_to_rp': {'type': 'ComputeResource', + 'path': 'resourcePool', + 'skip': False, + 'selectSet': [rp_to_rp_sel_spec, + rp_to_vm_sel_spec]}, + 'ccr_to_h': {'type': 'ClusterComputeResource', + 'path': 'host', + 'skip': False, + 'selectSet': []}, + 'ccr_to_ds': {'type': 'ClusterComputeResource', + 'path': 'datastore', + 'skip': False, + 'selectSet': []}, + 'ccr_to_rp': {'type': 'ClusterComputeResource', + 'path': 'resourcePool', + 'skip': False, + 'selectSet': [rp_to_rp_sel_spec, + rp_to_vm_sel_spec]}, + 'rp_to_rp': {'type': 'ResourcePool', + 'path': 'resourcePool', + 'skip': False, + 'selectSet': [rp_to_rp_sel_spec, + rp_to_vm_sel_spec]}, + 'rp_to_vm': {'type': 'ResourcePool', + 'path': 'vm', + 'skip': False, + 'selectSet': [rp_to_rp_sel_spec, + rp_to_vm_sel_spec]}, + } + + client_factory = mock.Mock() + client_factory.create.side_effect = lambda ns: mock.Mock() + trav_spec = vim_util.build_recursive_traversal_spec(client_factory) + self.assertEqual("visitFolders", trav_spec.name) + self.assertEqual("childEntity", trav_spec.path) + self.assertFalse(trav_spec.skip) + self.assertEqual("Folder", trav_spec.type) + + self.assertEqual(len(traversal_spec_dict) + 1, + len(trav_spec.selectSet)) + for spec in trav_spec.selectSet: + if spec.name not in traversal_spec_dict: + self.assertEqual(sel_spec, spec) + else: + exp_spec = traversal_spec_dict[spec.name] + self.assertEqual(exp_spec['type'], spec.type) + self.assertEqual(exp_spec['path'], spec.path) + self.assertEqual(exp_spec['skip'], spec.skip) + self.assertEqual(exp_spec['selectSet'], spec.selectSet) + + def test_build_property_spec(self): + client_factory = mock.Mock() + prop_spec = vim_util.build_property_spec(client_factory) + self.assertFalse(prop_spec.all) + self.assertEqual(["name"], prop_spec.pathSet) + self.assertEqual("VirtualMachine", prop_spec.type) + + def test_build_object_spec(self): + client_factory = mock.Mock() + root_folder = mock.Mock() + specs = [mock.Mock()] + obj_spec = vim_util.build_object_spec(client_factory, + root_folder, specs) + self.assertEqual(root_folder, obj_spec.obj) + self.assertEqual(specs, obj_spec.selectSet) + self.assertFalse(obj_spec.skip) + + def test_build_property_filter_spec(self): + client_factory = mock.Mock() + prop_specs = [mock.Mock()] + obj_specs = [mock.Mock()] + filter_spec = vim_util.build_property_filter_spec(client_factory, + prop_specs, + obj_specs) + self.assertEqual(obj_specs, filter_spec.objectSet) + self.assertEqual(prop_specs, filter_spec.propSet) + + @mock.patch( + 'oslo_vmware.vim_util.build_recursive_traversal_spec') + def test_get_objects(self, build_recursive_traversal_spec): + vim = mock.Mock() + trav_spec = mock.Mock() + build_recursive_traversal_spec.return_value = trav_spec + max_objects = 10 + _type = "VirtualMachine" + + def vim_RetrievePropertiesEx_side_effect(pc, specSet, options): + self.assertTrue(pc is vim.service_content.propertyCollector) + self.assertEqual(max_objects, options.maxObjects) + + self.assertEqual(1, len(specSet)) + property_filter_spec = specSet[0] + + propSet = property_filter_spec.propSet + self.assertEqual(1, len(propSet)) + prop_spec = propSet[0] + self.assertFalse(prop_spec.all) + self.assertEqual(["name"], prop_spec.pathSet) + self.assertEqual(_type, prop_spec.type) + + objSet = property_filter_spec.objectSet + self.assertEqual(1, len(objSet)) + obj_spec = objSet[0] + self.assertTrue(obj_spec.obj is vim.service_content.rootFolder) + self.assertEqual([trav_spec], obj_spec.selectSet) + self.assertFalse(obj_spec.skip) + + vim.RetrievePropertiesEx.side_effect = \ + vim_RetrievePropertiesEx_side_effect + vim_util.get_objects(vim, _type, max_objects) + self.assertEqual(1, vim.RetrievePropertiesEx.call_count) + + def test_get_object_properties_with_empty_moref(self): + vim = mock.Mock() + ret = vim_util.get_object_properties(vim, None, None) + self.assertIsNone(ret) + + @mock.patch('oslo_vmware.vim_util.cancel_retrieval') + def test_get_object_properties(self, cancel_retrieval): + vim = mock.Mock() + moref = mock.Mock() + moref._type = "VirtualMachine" + retrieve_result = mock.Mock() + + def vim_RetrievePropertiesEx_side_effect(pc, specSet, options): + self.assertTrue(pc is vim.service_content.propertyCollector) + self.assertEqual(1, options.maxObjects) + + self.assertEqual(1, len(specSet)) + property_filter_spec = specSet[0] + + propSet = property_filter_spec.propSet + self.assertEqual(1, len(propSet)) + prop_spec = propSet[0] + self.assertTrue(prop_spec.all) + self.assertEqual(['name'], prop_spec.pathSet) + self.assertEqual(moref._type, prop_spec.type) + + objSet = property_filter_spec.objectSet + self.assertEqual(1, len(objSet)) + obj_spec = objSet[0] + self.assertEqual(moref, obj_spec.obj) + self.assertEqual([], obj_spec.selectSet) + self.assertFalse(obj_spec.skip) + + return retrieve_result + + vim.RetrievePropertiesEx.side_effect = \ + vim_RetrievePropertiesEx_side_effect + + res = vim_util.get_object_properties(vim, moref, None) + self.assertEqual(1, vim.RetrievePropertiesEx.call_count) + self.assertTrue(res is retrieve_result.objects) + cancel_retrieval.assert_called_once_with(vim, retrieve_result) + + def test_get_token(self): + retrieve_result = object() + self.assertFalse(vim_util._get_token(retrieve_result)) + + @mock.patch('oslo_vmware.vim_util._get_token') + def test_cancel_retrieval(self, get_token): + token = mock.Mock() + get_token.return_value = token + vim = mock.Mock() + retrieve_result = mock.Mock() + vim_util.cancel_retrieval(vim, retrieve_result) + get_token.assert_called_once_with(retrieve_result) + vim.CancelRetrievePropertiesEx.assert_called_once_with( + vim.service_content.propertyCollector, token=token) + + @mock.patch('oslo_vmware.vim_util._get_token') + def test_continue_retrieval(self, get_token): + token = mock.Mock() + get_token.return_value = token + vim = mock.Mock() + retrieve_result = mock.Mock() + vim_util.continue_retrieval(vim, retrieve_result) + get_token.assert_called_once_with(retrieve_result) + vim.ContinueRetrievePropertiesEx.assert_called_once_with( + vim.service_content.propertyCollector, token=token) + + @mock.patch('oslo_vmware.vim_util.get_object_properties') + def test_get_object_property(self, get_object_properties): + prop = mock.Mock() + prop.val = "ubuntu-12.04" + properties = mock.Mock() + properties.propSet = [prop] + properties_list = [properties] + get_object_properties.return_value = properties_list + vim = mock.Mock() + moref = mock.Mock() + property_name = 'name' + val = vim_util.get_object_property(vim, moref, property_name) + self.assertEqual(prop.val, val) + get_object_properties.assert_called_once_with( + vim, moref, [property_name]) + + def test_find_extension(self): + vim = mock.Mock() + ret = vim_util.find_extension(vim, 'fake-key') + self.assertIsNotNone(ret) + service_content = vim.service_content + vim.client.service.FindExtension.assert_called_once_with( + service_content.extensionManager, 'fake-key') + + def test_register_extension(self): + vim = mock.Mock() + ret = vim_util.register_extension(vim, 'fake-key', 'fake-type') + self.assertIsNone(ret) + service_content = vim.service_content + vim.client.service.RegisterExtension.assert_called_once_with( + service_content.extensionManager, mock.ANY) + + def test_get_vc_version(self): + session = mock.Mock() + expected_version = '6.0.1' + session.vim.service_content.about.version = expected_version + version = vim_util.get_vc_version(session) + self.assertEqual(expected_version, version) + expected_version = '5.5' + session.vim.service_content.about.version = expected_version + version = vim_util.get_vc_version(session) + self.assertEqual(expected_version, version) + + def test_get_inventory_path_folders(self): + ObjectContent = collections.namedtuple('ObjectContent', ['propSet']) + DynamicProperty = collections.namedtuple('Property', ['name', 'val']) + + obj1 = ObjectContent(propSet=[ + DynamicProperty(name='Datacenter', val='dc-1'), + ]) + obj2 = ObjectContent(propSet=[ + DynamicProperty(name='Datacenter', val='folder-2'), + ]) + obj3 = ObjectContent(propSet=[ + DynamicProperty(name='Datacenter', val='folder-1'), + ]) + objects = ['foo', 'bar', obj1, obj2, obj3] + result = mock.sentinel.objects + result.objects = objects + session = mock.Mock() + session.vim.RetrievePropertiesEx = mock.Mock() + session.vim.RetrievePropertiesEx.return_value = result + entity = mock.Mock() + inv_path = vim_util.get_inventory_path(session.vim, entity, 100) + self.assertEqual('/folder-2/dc-1', inv_path) + + def test_get_inventory_path_no_folder(self): + ObjectContent = collections.namedtuple('ObjectContent', ['propSet']) + DynamicProperty = collections.namedtuple('Property', ['name', 'val']) + + obj1 = ObjectContent(propSet=[ + DynamicProperty(name='Datacenter', val='dc-1'), + ]) + objects = ['foo', 'bar', obj1] + result = mock.sentinel.objects + result.objects = objects + session = mock.Mock() + session.vim.RetrievePropertiesEx = mock.Mock() + session.vim.RetrievePropertiesEx.return_value = result + entity = mock.Mock() + inv_path = vim_util.get_inventory_path(session.vim, entity, 100) + self.assertEqual('dc-1', inv_path) diff --git a/oslo_vmware/vim.py b/oslo_vmware/vim.py new file mode 100644 index 00000000..8343de65 --- /dev/null +++ b/oslo_vmware/vim.py @@ -0,0 +1,50 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_vmware import service + + +class Vim(service.Service): + """Service class that provides access to the VIM API.""" + + def __init__(self, protocol='https', host='localhost', port=None, + wsdl_url=None, cacert=None, insecure=True): + """Constructs a VIM service client object. + + :param protocol: http or https + :param host: server IP address or host name + :param port: port for connection + :param wsdl_url: VIM WSDL url + :param cacert: Specify a CA bundle file to use in verifying a + TLS (https) server certificate. + :param insecure: Verify HTTPS connections using system certificates, + used only if cacert is not specified + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + base_url = service.Service.build_base_url(protocol, host, port) + soap_url = base_url + '/sdk' + if wsdl_url is None: + wsdl_url = soap_url + '/vimService.wsdl' + super(Vim, self).__init__(wsdl_url, soap_url, cacert, insecure) + + def retrieve_service_content(self): + return self.RetrieveServiceContent(service.SERVICE_INSTANCE) + + def __repr__(self): + return "VIM Object" + + def __str__(self): + return "VIM Object" diff --git a/oslo_vmware/vim_util.py b/oslo_vmware/vim_util.py new file mode 100644 index 00000000..fd43ab56 --- /dev/null +++ b/oslo_vmware/vim_util.py @@ -0,0 +1,486 @@ +# Copyright (c) 2014 VMware, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +The VMware API utility module. +""" + +from suds import sudsobject + +from oslo.utils import timeutils + + +def get_moref(value, type_): + """Get managed object reference. + + :param value: value of the managed object + :param type_: type of the managed object + :returns: managed object reference with given value and type + """ + moref = sudsobject.Property(value) + moref._type = type_ + return moref + + +def build_selection_spec(client_factory, name): + """Builds the selection spec. + + :param client_factory: factory to get API input specs + :param name: name for the selection spec + :returns: selection spec + """ + sel_spec = client_factory.create('ns0:SelectionSpec') + sel_spec.name = name + return sel_spec + + +def build_traversal_spec(client_factory, name, type_, path, skip, select_set): + """Builds the traversal spec. + + :param client_factory: factory to get API input specs + :param name: name for the traversal spec + :param type_: type of the managed object + :param path: property path of the managed object + :param skip: whether or not to filter the object identified by param path + :param select_set: set of selection specs specifying additional objects + to filter + :returns: traversal spec + """ + traversal_spec = client_factory.create('ns0:TraversalSpec') + traversal_spec.name = name + traversal_spec.type = type_ + traversal_spec.path = path + traversal_spec.skip = skip + traversal_spec.selectSet = select_set + return traversal_spec + + +def build_recursive_traversal_spec(client_factory): + """Builds recursive traversal spec to traverse managed object hierarchy. + + :param client_factory: factory to get API input specs + :returns: recursive traversal spec + """ + visit_folders_select_spec = build_selection_spec(client_factory, + 'visitFolders') + # Next hop from Datacenter + dc_to_hf = build_traversal_spec(client_factory, + 'dc_to_hf', + 'Datacenter', + 'hostFolder', + False, + [visit_folders_select_spec]) + dc_to_vmf = build_traversal_spec(client_factory, + 'dc_to_vmf', + 'Datacenter', + 'vmFolder', + False, + [visit_folders_select_spec]) + dc_to_netf = build_traversal_spec(client_factory, + 'dc_to_netf', + 'Datacenter', + 'networkFolder', + False, + [visit_folders_select_spec]) + + # Next hop from HostSystem + h_to_vm = build_traversal_spec(client_factory, + 'h_to_vm', + 'HostSystem', + 'vm', + False, + [visit_folders_select_spec]) + + # Next hop from ComputeResource + cr_to_h = build_traversal_spec(client_factory, + 'cr_to_h', + 'ComputeResource', + 'host', + False, + []) + cr_to_ds = build_traversal_spec(client_factory, + 'cr_to_ds', + 'ComputeResource', + 'datastore', + False, + []) + + rp_to_rp_select_spec = build_selection_spec(client_factory, 'rp_to_rp') + rp_to_vm_select_spec = build_selection_spec(client_factory, 'rp_to_vm') + + cr_to_rp = build_traversal_spec(client_factory, + 'cr_to_rp', + 'ComputeResource', + 'resourcePool', + False, + [rp_to_rp_select_spec, + rp_to_vm_select_spec]) + + # Next hop from ClusterComputeResource + ccr_to_h = build_traversal_spec(client_factory, + 'ccr_to_h', + 'ClusterComputeResource', + 'host', + False, + []) + ccr_to_ds = build_traversal_spec(client_factory, + 'ccr_to_ds', + 'ClusterComputeResource', + 'datastore', + False, + []) + ccr_to_rp = build_traversal_spec(client_factory, + 'ccr_to_rp', + 'ClusterComputeResource', + 'resourcePool', + False, + [rp_to_rp_select_spec, + rp_to_vm_select_spec]) + # Next hop from ResourcePool + rp_to_rp = build_traversal_spec(client_factory, + 'rp_to_rp', + 'ResourcePool', + 'resourcePool', + False, + [rp_to_rp_select_spec, + rp_to_vm_select_spec]) + rp_to_vm = build_traversal_spec(client_factory, + 'rp_to_vm', + 'ResourcePool', + 'vm', + False, + [rp_to_rp_select_spec, + rp_to_vm_select_spec]) + + # Get the assorted traversal spec which takes care of the objects to + # be searched for from the rootFolder + traversal_spec = build_traversal_spec(client_factory, + 'visitFolders', + 'Folder', + 'childEntity', + False, + [visit_folders_select_spec, + h_to_vm, + dc_to_hf, + dc_to_vmf, + dc_to_netf, + cr_to_ds, + cr_to_h, + cr_to_rp, + ccr_to_h, + ccr_to_ds, + ccr_to_rp, + rp_to_rp, + rp_to_vm]) + return traversal_spec + + +def build_property_spec(client_factory, type_='VirtualMachine', + properties_to_collect=None, all_properties=False): + """Builds the property spec. + + :param client_factory: factory to get API input specs + :param type_: type of the managed object + :param properties_to_collect: names of the managed object properties to be + collected while traversal filtering + :param all_properties: whether all properties of the managed object need + to be collected + :returns: property spec + """ + if not properties_to_collect: + properties_to_collect = ['name'] + + property_spec = client_factory.create('ns0:PropertySpec') + property_spec.all = all_properties + property_spec.pathSet = properties_to_collect + property_spec.type = type_ + return property_spec + + +def build_object_spec(client_factory, root_folder, traversal_specs): + """Builds the object spec. + + :param client_factory: factory to get API input specs + :param root_folder: root folder reference; the starting point of traversal + :param traversal_specs: filter specs required for traversal + :returns: object spec + """ + object_spec = client_factory.create('ns0:ObjectSpec') + object_spec.obj = root_folder + object_spec.skip = False + object_spec.selectSet = traversal_specs + return object_spec + + +def build_property_filter_spec(client_factory, property_specs, object_specs): + """Builds the property filter spec. + + :param client_factory: factory to get API input specs + :param property_specs: property specs to be collected for filtered objects + :param object_specs: object specs to identify objects to be filtered + :returns: property filter spec + """ + property_filter_spec = client_factory.create('ns0:PropertyFilterSpec') + property_filter_spec.propSet = property_specs + property_filter_spec.objectSet = object_specs + return property_filter_spec + + +def get_objects(vim, type_, max_objects, properties_to_collect=None, + all_properties=False): + """Get all managed object references of the given type. + + It is the caller's responsibility to continue or cancel retrieval. + + :param vim: Vim object + :param type_: type of the managed object + :param max_objects: maximum number of objects that should be returned in + a single call + :param properties_to_collect: names of the managed object properties to be + collected + :param all_properties: whether all properties of the managed object need to + be collected + :returns: all managed object references of the given type + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + if not properties_to_collect: + properties_to_collect = ['name'] + + client_factory = vim.client.factory + recur_trav_spec = build_recursive_traversal_spec(client_factory) + object_spec = build_object_spec(client_factory, + vim.service_content.rootFolder, + [recur_trav_spec]) + property_spec = build_property_spec( + client_factory, + type_=type_, + properties_to_collect=properties_to_collect, + all_properties=all_properties) + property_filter_spec = build_property_filter_spec(client_factory, + [property_spec], + [object_spec]) + options = client_factory.create('ns0:RetrieveOptions') + options.maxObjects = max_objects + return vim.RetrievePropertiesEx(vim.service_content.propertyCollector, + specSet=[property_filter_spec], + options=options) + + +def get_object_properties(vim, moref, properties_to_collect): + """Get properties of the given managed object. + + :param vim: Vim object + :param moref: managed object reference + :param properties_to_collect: names of the managed object properties to be + collected + :returns: properties of the given managed object + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + if moref is None: + return None + + client_factory = vim.client.factory + all_properties = (properties_to_collect is None or + len(properties_to_collect) == 0) + property_spec = build_property_spec( + client_factory, + type_=moref._type, + properties_to_collect=properties_to_collect, + all_properties=all_properties) + object_spec = build_object_spec(client_factory, moref, []) + property_filter_spec = build_property_filter_spec(client_factory, + [property_spec], + [object_spec]) + + options = client_factory.create('ns0:RetrieveOptions') + options.maxObjects = 1 + retrieve_result = vim.RetrievePropertiesEx( + vim.service_content.propertyCollector, + specSet=[property_filter_spec], + options=options) + cancel_retrieval(vim, retrieve_result) + return retrieve_result.objects + + +def _get_token(retrieve_result): + """Get token from result to obtain next set of results. + + :retrieve_result: Result of RetrievePropertiesEx API call + :returns: token to obtain next set of results; None if no more results. + """ + return getattr(retrieve_result, 'token', None) + + +def cancel_retrieval(vim, retrieve_result): + """Cancels the retrieve operation if necessary. + + :param vim: Vim object + :param retrieve_result: result of RetrievePropertiesEx API call + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + token = _get_token(retrieve_result) + if token: + collector = vim.service_content.propertyCollector + vim.CancelRetrievePropertiesEx(collector, token=token) + + +def continue_retrieval(vim, retrieve_result): + """Continue retrieving results, if available. + + :param vim: Vim object + :param retrieve_result: result of RetrievePropertiesEx API call + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + token = _get_token(retrieve_result) + if token: + collector = vim.service_content.propertyCollector + return vim.ContinueRetrievePropertiesEx(collector, token=token) + + +def get_object_property(vim, moref, property_name): + """Get property of the given managed object. + + :param vim: Vim object + :param moref: managed object reference + :param property_name: name of the property to be retrieved + :returns: property of the given managed object + :raises: VimException, VimFaultException, VimAttributeException, + VimSessionOverLoadException, VimConnectionException + """ + props = get_object_properties(vim, moref, [property_name]) + prop_val = None + if props: + prop = None + if hasattr(props[0], 'propSet'): + # propSet will be set only if the server provides value + # for the field + prop = props[0].propSet + if prop: + prop_val = prop[0].val + return prop_val + + +def find_extension(vim, key): + """Looks for an existing extension. + + :param vim: Vim object + :param key: the key to search for + :returns: the data object Extension or None + """ + extension_manager = vim.service_content.extensionManager + return vim.client.service.FindExtension(extension_manager, key) + + +def register_extension(vim, key, type, label='OpenStack', + summary='OpenStack services', version='1.0'): + """Create a new extention. + + :param vim: Vim object + :param key: the key for the extension + :param type: Managed entity type, as defined by the extension. This + matches the type field in the configuration about a + virtual machine or vApp + :param label: Display label + :param summary: Summary description + :param version: Extension version number as a dot-separated string + """ + extension_manager = vim.service_content.extensionManager + client_factory = vim.client.factory + os_ext = client_factory.create('ns0:Extension') + os_ext.key = key + entity_info = client_factory.create('ns0:ExtManagedEntityInfo') + entity_info.type = type + os_ext.managedEntityInfo = [entity_info] + os_ext.version = version + desc = client_factory.create('ns0:Description') + desc.label = label + desc.summary = summary + os_ext.description = desc + os_ext.lastHeartbeatTime = timeutils.strtime() + vim.client.service.RegisterExtension(extension_manager, os_ext) + + +def get_vc_version(session): + """Return the dot-separated vCenter version string. For example, "1.2". + + :param session: vCenter soap session + :return: vCenter version + """ + return session.vim.service_content.about.version + + +def get_inventory_path(vim, entity_ref, max_objects=100): + """Get the inventory path of a managed entity. + + :param vim: Vim object + :param entity_ref: managed entity reference + :param max_objects: maximum number of objects that should be returned in + a single call + :return: inventory path of the entity_ref + """ + client_factory = vim.client.factory + property_collector = vim.service_content.propertyCollector + + prop_spec = build_property_spec(client_factory, 'ManagedEntity', + ['name', 'parent']) + select_set = build_selection_spec(client_factory, 'ParentTraversalSpec') + select_set = build_traversal_spec( + client_factory, 'ParentTraversalSpec', 'ManagedEntity', 'parent', + False, [select_set]) + obj_spec = build_object_spec(client_factory, entity_ref, select_set) + prop_filter_spec = build_property_filter_spec(client_factory, + [prop_spec], [obj_spec]) + options = client_factory.create('ns0:RetrieveOptions') + options.maxObjects = max_objects + retrieve_result = vim.RetrievePropertiesEx( + property_collector, + specSet=[prop_filter_spec], + options=options) + entity_name = None + propSet = None + path = "" + while retrieve_result: + for obj in retrieve_result.objects: + if hasattr(obj, 'propSet'): + propSet = obj.propSet + if len(propSet) >= 1 and not entity_name: + entity_name = propSet[0].val + elif len(propSet) >= 1: + path = '%s/%s' % (propSet[0].val, path) + retrieve_result = continue_retrieval(vim, retrieve_result) + # NOTE(arnaud): slice to exclude the root folder from the result. + if propSet is not None and len(propSet) > 0: + path = path[len(propSet[0].val):] + if entity_name is None: + entity_name = "" + return '%s%s' % (path, entity_name) + + +def get_http_service_request_spec(client_factory, method, uri): + """Build a HTTP service request spec. + + :param client_factory: factory to get API input specs + :param method: HTTP method (GET, POST, PUT) + :param uri: target URL + """ + http_service_request_spec = client_factory.create( + 'ns0:SessionManagerHttpServiceRequestSpec') + http_service_request_spec.method = method + http_service_request_spec.url = uri + return http_service_request_spec diff --git a/oslo/vmware/wsdl/5.5/core-types.xsd b/oslo_vmware/wsdl/5.5/core-types.xsd similarity index 100% rename from oslo/vmware/wsdl/5.5/core-types.xsd rename to oslo_vmware/wsdl/5.5/core-types.xsd diff --git a/oslo/vmware/wsdl/5.5/pbm-messagetypes.xsd b/oslo_vmware/wsdl/5.5/pbm-messagetypes.xsd similarity index 100% rename from oslo/vmware/wsdl/5.5/pbm-messagetypes.xsd rename to oslo_vmware/wsdl/5.5/pbm-messagetypes.xsd diff --git a/oslo/vmware/wsdl/5.5/pbm-types.xsd b/oslo_vmware/wsdl/5.5/pbm-types.xsd similarity index 100% rename from oslo/vmware/wsdl/5.5/pbm-types.xsd rename to oslo_vmware/wsdl/5.5/pbm-types.xsd diff --git a/oslo/vmware/wsdl/5.5/pbm.wsdl b/oslo_vmware/wsdl/5.5/pbm.wsdl similarity index 100% rename from oslo/vmware/wsdl/5.5/pbm.wsdl rename to oslo_vmware/wsdl/5.5/pbm.wsdl diff --git a/oslo/vmware/wsdl/5.5/pbmService.wsdl b/oslo_vmware/wsdl/5.5/pbmService.wsdl similarity index 100% rename from oslo/vmware/wsdl/5.5/pbmService.wsdl rename to oslo_vmware/wsdl/5.5/pbmService.wsdl diff --git a/setup.cfg b/setup.cfg index 23ad290c..29e651a6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifier = [files] packages = oslo + oslo_vmware namespace_packages = oslo diff --git a/tests/objects/test_datastore.py b/tests/objects/test_datastore.py index b8e3c1f0..2d66ccff 100644 --- a/tests/objects/test_datastore.py +++ b/tests/objects/test_datastore.py @@ -19,6 +19,7 @@ from oslo.utils import units from oslo.vmware import constants from oslo.vmware.objects import datastore from oslo.vmware import vim_util +from oslo_vmware import vim_util as new_vim_util from tests import base @@ -89,7 +90,7 @@ class DatastoreTestCase(base.TestCase): session.invoke_api.return_value = summary ret = ds.get_summary(session) self.assertEqual(summary, ret) - session.invoke_api.assert_called_once_with(vim_util, + session.invoke_api.assert_called_once_with(new_vim_util, 'get_object_property', session.vim, ds.ref, 'summary') diff --git a/tests/test_api.py b/tests/test_api.py index 6cccc169..78b7e382 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -25,8 +25,7 @@ import suds from oslo.vmware import api from oslo.vmware import exceptions -from oslo.vmware import pbm -from oslo.vmware import vim_util +from oslo_vmware import vim_util as new_vim_util from tests import base @@ -105,7 +104,7 @@ class VMwareAPISessionTest(base.TestCase): def setUp(self): super(VMwareAPISessionTest, self).setUp() - patcher = mock.patch('oslo.vmware.vim.Vim') + patcher = mock.patch('oslo_vmware.vim.Vim') self.addCleanup(patcher.stop) self.VimMock = patcher.start() self.VimMock.side_effect = lambda *args, **kw: mock.MagicMock() @@ -134,7 +133,7 @@ class VMwareAPISessionTest(base.TestCase): cacert=self.cert_mock, insecure=False) - @mock.patch.object(pbm, 'Pbm') + @mock.patch('oslo_vmware.pbm.Pbm') def test_pbm(self, pbm_mock): api_session = self._create_api_session(True) vim_obj = api_session.vim @@ -372,7 +371,7 @@ class VMwareAPISessionTest(base.TestCase): ret = api_session.wait_for_task(task) self.assertEqual('success', ret.state) self.assertEqual(100, ret.progress) - api_session.invoke_api.assert_called_with(vim_util, + api_session.invoke_api.assert_called_with(new_vim_util, 'get_object_property', api_session.vim, task, 'info') @@ -397,7 +396,7 @@ class VMwareAPISessionTest(base.TestCase): self.assertRaises(exceptions.VMwareDriverException, api_session.wait_for_task, task) - api_session.invoke_api.assert_called_with(vim_util, + api_session.invoke_api.assert_called_with(new_vim_util, 'get_object_property', api_session.vim, task, 'info') @@ -413,7 +412,7 @@ class VMwareAPISessionTest(base.TestCase): self.assertRaises(exceptions.VimException, api_session.wait_for_task, task) - api_session.invoke_api.assert_called_once_with(vim_util, + api_session.invoke_api.assert_called_once_with(new_vim_util, 'get_object_property', api_session.vim, task, 'info') @@ -430,7 +429,7 @@ class VMwareAPISessionTest(base.TestCase): lease = mock.Mock() with mock.patch.object(greenthread, 'sleep'): api_session.wait_for_lease_ready(lease) - api_session.invoke_api.assert_called_with(vim_util, + api_session.invoke_api.assert_called_with(new_vim_util, 'get_object_property', api_session.vim, lease, 'state') @@ -449,9 +448,9 @@ class VMwareAPISessionTest(base.TestCase): self.assertRaises(exceptions.VimException, api_session.wait_for_lease_ready, lease) - exp_calls = [mock.call(vim_util, 'get_object_property', + exp_calls = [mock.call(new_vim_util, 'get_object_property', api_session.vim, lease, 'state')] * 2 - exp_calls.append(mock.call(vim_util, 'get_object_property', + exp_calls.append(mock.call(new_vim_util, 'get_object_property', api_session.vim, lease, 'error')) self.assertEqual(exp_calls, api_session.invoke_api.call_args_list) @@ -466,7 +465,7 @@ class VMwareAPISessionTest(base.TestCase): self.assertRaises(exceptions.VimException, api_session.wait_for_lease_ready, lease) - api_session.invoke_api.assert_called_once_with(vim_util, + api_session.invoke_api.assert_called_once_with(new_vim_util, 'get_object_property', api_session.vim, lease, 'state') @@ -480,7 +479,7 @@ class VMwareAPISessionTest(base.TestCase): api_session.wait_for_lease_ready, lease) api_session.invoke_api.assert_called_once_with( - vim_util, 'get_object_property', api_session.vim, lease, + new_vim_util, 'get_object_property', api_session.vim, lease, 'state') def _poll_task_well_known_exceptions(self, fault, @@ -506,10 +505,6 @@ class VMwareAPISessionTest(base.TestCase): api_session._poll_task, 'fake-task') - def test_poll_task_well_known_exceptions(self): - for k, v in six.iteritems(exceptions._fault_classes_registry): - self._poll_task_well_known_exceptions(k, v) - def test_poll_task_unknown_exception(self): _unknown_exceptions = { 'NoDiskSpace': exceptions.VMwareDriverException, diff --git a/tests/test_image_transfer.py b/tests/test_image_transfer.py index 1f613d03..91f71f5b 100644 --- a/tests/test_image_transfer.py +++ b/tests/test_image_transfer.py @@ -26,6 +26,7 @@ import mock from oslo.vmware import exceptions from oslo.vmware import image_transfer from oslo.vmware import rw_handles +from oslo_vmware import image_transfer as new_image_transfer from tests import base @@ -191,9 +192,9 @@ class ImageTransferUtilityTest(base.TestCase): """Tests for image_transfer utility methods.""" @mock.patch.object(timeout, 'Timeout') - @mock.patch.object(image_transfer, 'ImageWriter') - @mock.patch.object(image_transfer, 'FileReadWriteTask') - @mock.patch.object(image_transfer, 'BlockingQueue') + @mock.patch('oslo_vmware.image_transfer.ImageWriter') + @mock.patch('oslo_vmware.image_transfer.FileReadWriteTask') + @mock.patch('oslo_vmware.image_transfer.BlockingQueue') def test_start_transfer(self, fake_BlockingQueue, fake_FileReadWriteTask, fake_ImageWriter, fake_Timeout): @@ -220,14 +221,15 @@ class ImageTransferUtilityTest(base.TestCase): fake_Timeout.return_value = fake_timer for write_file_handle in write_file_handles: - image_transfer._start_transfer(context, - timeout_secs, - read_file_handle, - max_data_size, - write_file_handle=write_file_handle, - image_service=image_service, - image_id=image_id, - image_meta=image_meta) + new_image_transfer._start_transfer( + context, + timeout_secs, + read_file_handle, + max_data_size, + write_file_handle=write_file_handle, + image_service=image_service, + image_id=image_id, + image_meta=image_meta) exp_calls = [mock.call(blocking_queue_size, max_data_size)] * len(write_file_handles) @@ -257,44 +259,9 @@ class ImageTransferUtilityTest(base.TestCase): write_file_handle1.close.assert_called_once() - @mock.patch.object(image_transfer, 'FileReadWriteTask') - @mock.patch.object(image_transfer, 'BlockingQueue') - def test_start_transfer_with_no_image_destination(self, fake_BlockingQueue, - fake_FileReadWriteTask): - - context = mock.Mock() - read_file_handle = mock.Mock() - write_file_handle = None - image_service = None - image_id = None - timeout_secs = 10 - image_meta = {} - blocking_queue_size = 10 - max_data_size = 30 - blocking_queue = mock.Mock() - - fake_BlockingQueue.return_value = blocking_queue - - self.assertRaises(ValueError, - image_transfer._start_transfer, - context, - timeout_secs, - read_file_handle, - max_data_size, - write_file_handle=write_file_handle, - image_service=image_service, - image_id=image_id, - image_meta=image_meta) - - fake_BlockingQueue.assert_called_once_with(blocking_queue_size, - max_data_size) - - fake_FileReadWriteTask.assert_called_once_with(read_file_handle, - blocking_queue) - - @mock.patch('oslo.vmware.rw_handles.FileWriteHandle') - @mock.patch('oslo.vmware.rw_handles.ImageReadHandle') - @mock.patch.object(image_transfer, '_start_transfer') + @mock.patch('oslo_vmware.rw_handles.FileWriteHandle') + @mock.patch('oslo_vmware.rw_handles.ImageReadHandle') + @mock.patch('oslo_vmware.image_transfer._start_transfer') def test_download_flat_image( self, fake_transfer, @@ -355,8 +322,8 @@ class ImageTransferUtilityTest(base.TestCase): image_size, write_file_handle=fake_FileWriteHandle) - @mock.patch('oslo.vmware.rw_handles.VmdkWriteHandle') - @mock.patch.object(image_transfer, '_start_transfer') + @mock.patch('oslo_vmware.rw_handles.VmdkWriteHandle') + @mock.patch('oslo_vmware.image_transfer._start_transfer') def test_download_stream_optimized_data(self, fake_transfer, fake_rw_handles_VmdkWriteHandle): @@ -405,8 +372,8 @@ class ImageTransferUtilityTest(base.TestCase): fake_VmdkWriteHandle.get_imported_vm.assert_called_once() - @mock.patch('oslo.vmware.rw_handles.ImageReadHandle') - @mock.patch.object(image_transfer, 'download_stream_optimized_data') + @mock.patch('oslo_vmware.rw_handles.ImageReadHandle') + @mock.patch('oslo_vmware.image_transfer.download_stream_optimized_data') def test_download_stream_optimized_image( self, fake_download_stream_optimized_data, fake_rw_handles_ImageReadHandle): @@ -459,8 +426,8 @@ class ImageTransferUtilityTest(base.TestCase): vm_import_spec=vm_import_spec, image_size=image_size) - @mock.patch.object(image_transfer, '_start_transfer') - @mock.patch('oslo.vmware.rw_handles.VmdkReadHandle') + @mock.patch('oslo_vmware.image_transfer._start_transfer') + @mock.patch('oslo_vmware.rw_handles.VmdkReadHandle') def test_copy_stream_optimized_disk( self, vmdk_read_handle, start_transfer): @@ -488,8 +455,8 @@ class ImageTransferUtilityTest(base.TestCase): context, timeout, read_handle, vmdk_size, write_file_handle=write_handle) - @mock.patch('oslo.vmware.rw_handles.VmdkReadHandle') - @mock.patch.object(image_transfer, '_start_transfer') + @mock.patch('oslo_vmware.rw_handles.VmdkReadHandle') + @mock.patch('oslo_vmware.image_transfer._start_transfer') def test_upload_image(self, fake_transfer, fake_rw_handles_VmdkReadHandle): context = mock.Mock() diff --git a/tests/test_pbm.py b/tests/test_pbm.py index 24fdd99d..ac60d4a2 100644 --- a/tests/test_pbm.py +++ b/tests/test_pbm.py @@ -24,6 +24,7 @@ import six.moves.urllib.parse as urlparse import six.moves.urllib.request as urllib from oslo.vmware import pbm +from oslo_vmware import pbm as new_pbm from tests import base @@ -69,7 +70,7 @@ class PBMUtilityTest(base.TestCase): profile.name = name return profile - @mock.patch.object(pbm, 'get_all_profiles') + @mock.patch('oslo_vmware.pbm.get_all_profiles') def test_get_profile_id_by_name(self, get_all_profiles): profiles = [self._create_profile(str(i), 'profile-%d' % i) for i in range(0, 10)] @@ -82,7 +83,7 @@ class PBMUtilityTest(base.TestCase): self.assertEqual(exp_profile_id, profile_id) get_all_profiles.assert_called_once_with(session) - @mock.patch.object(pbm, 'get_all_profiles') + @mock.patch('oslo_vmware.pbm.get_all_profiles') def test_get_profile_id_by_name_with_invalid_profile(self, get_all_profiles): profiles = [self._create_profile(str(i), 'profile-%d' % i) @@ -155,7 +156,7 @@ class PBMUtilityTest(base.TestCase): self.assertIsNone(wsdl) def expected_wsdl(version): - driver_abs_dir = os.path.abspath(os.path.dirname(pbm.__file__)) + driver_abs_dir = os.path.abspath(os.path.dirname(new_pbm.__file__)) path = os.path.join(driver_abs_dir, 'wsdl', version, 'pbmService.wsdl') return urlparse.urljoin('file:', urllib.pathname2url(path)) diff --git a/tests/test_rw_handles.py b/tests/test_rw_handles.py index 39a981ef..edccbb6d 100644 --- a/tests/test_rw_handles.py +++ b/tests/test_rw_handles.py @@ -22,7 +22,7 @@ import six from oslo.vmware import exceptions from oslo.vmware import rw_handles -from oslo.vmware import vim_util +from oslo_vmware import vim_util as new_vim_util from tests import base @@ -166,7 +166,7 @@ class VmdkWriteHandleTest(base.TestCase): 100) def session_invoke_api_side_effect(module, method, *args, **kwargs): - if module == vim_util and method == 'get_object_property': + if module == new_vim_util and method == 'get_object_property': return 'ready' self.assertEqual(session.vim, module) self.assertEqual('HttpNfcLeaseComplete', method) @@ -262,7 +262,7 @@ class VmdkReadHandleTest(base.TestCase): 100) def session_invoke_api_side_effect(module, method, *args, **kwargs): - if module == vim_util and method == 'get_object_property': + if module == new_vim_util and method == 'get_object_property': return 'ready' self.assertEqual(session.vim, module) self.assertEqual('HttpNfcLeaseComplete', method) diff --git a/tests/test_service.py b/tests/test_service.py index df4ff893..c80ceab6 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -21,7 +21,6 @@ import suds from oslo.vmware import exceptions from oslo.vmware import service -from oslo.vmware import vim_util from tests import base @@ -287,7 +286,7 @@ class ServiceTest(base.TestCase): svc_obj.powerOn, managed_object) - @mock.patch.object(vim_util, 'get_moref', return_value=None) + @mock.patch('oslo_vmware.vim_util.get_moref', return_value=None) def test_request_handler_no_value(self, mock_moref): managed_object = 'VirtualMachine' svc_obj = service.Service() diff --git a/tests/test_vim.py b/tests/test_vim.py index 0c8aa513..6050bcea 100644 --- a/tests/test_vim.py +++ b/tests/test_vim.py @@ -20,9 +20,9 @@ Unit tests for classes to invoke VMware VI SOAP calls. import mock from oslo_i18n import fixture as i18n_fixture -from oslo.vmware._i18n import _ from oslo.vmware import exceptions from oslo.vmware import vim +from oslo_vmware._i18n import _ from tests import base diff --git a/tests/test_vim_util.py b/tests/test_vim_util.py index 3e1eb9a4..44b65267 100644 --- a/tests/test_vim_util.py +++ b/tests/test_vim_util.py @@ -52,7 +52,7 @@ class VimUtilTest(base.TestCase): self.assertFalse(traversal_spec.skip) self.assertEqual("Datacenter", traversal_spec.type) - @mock.patch.object(vim_util, 'build_selection_spec') + @mock.patch('oslo_vmware.vim_util.build_selection_spec') def test_build_recursive_traversal_spec(self, build_selection_spec_mock): sel_spec = mock.Mock() rp_to_rp_sel_spec = mock.Mock() @@ -176,7 +176,7 @@ class VimUtilTest(base.TestCase): self.assertEqual(prop_specs, filter_spec.propSet) @mock.patch( - 'oslo.vmware.vim_util.build_recursive_traversal_spec') + 'oslo_vmware.vim_util.build_recursive_traversal_spec') def test_get_objects(self, build_recursive_traversal_spec): vim = mock.Mock() trav_spec = mock.Mock() @@ -215,7 +215,7 @@ class VimUtilTest(base.TestCase): ret = vim_util.get_object_properties(vim, None, None) self.assertIsNone(ret) - @mock.patch('oslo.vmware.vim_util.cancel_retrieval') + @mock.patch('oslo_vmware.vim_util.cancel_retrieval') def test_get_object_properties(self, cancel_retrieval): vim = mock.Mock() moref = mock.Mock() @@ -253,11 +253,7 @@ class VimUtilTest(base.TestCase): self.assertTrue(res is retrieve_result.objects) cancel_retrieval.assert_called_once_with(vim, retrieve_result) - def test_get_token(self): - retrieve_result = object() - self.assertFalse(vim_util._get_token(retrieve_result)) - - @mock.patch('oslo.vmware.vim_util._get_token') + @mock.patch('oslo_vmware.vim_util._get_token') def test_cancel_retrieval(self, get_token): token = mock.Mock() get_token.return_value = token @@ -268,7 +264,7 @@ class VimUtilTest(base.TestCase): vim.CancelRetrievePropertiesEx.assert_called_once_with( vim.service_content.propertyCollector, token=token) - @mock.patch('oslo.vmware.vim_util._get_token') + @mock.patch('oslo_vmware.vim_util._get_token') def test_continue_retrieval(self, get_token): token = mock.Mock() get_token.return_value = token @@ -279,7 +275,7 @@ class VimUtilTest(base.TestCase): vim.ContinueRetrievePropertiesEx.assert_called_once_with( vim.service_content.propertyCollector, token=token) - @mock.patch('oslo.vmware.vim_util.get_object_properties') + @mock.patch('oslo_vmware.vim_util.get_object_properties') def test_get_object_property(self, get_object_properties): prop = mock.Mock() prop.val = "ubuntu-12.04" diff --git a/tox.ini b/tox.ini index f5905616..7fcf2d7f 100644 --- a/tox.ini +++ b/tox.ini @@ -38,5 +38,6 @@ ignore = H405,H904 exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,__init__.py [hacking] -import_exceptions = oslo.vmware._i18n +import_exceptions = oslo_vmware._i18n + oslo_vmware.tests.base tests.base