From f2ddd30737e57cdeca2c7bb7b52f3a65b2202a7e Mon Sep 17 00:00:00 2001 From: jason bishop Date: Thu, 22 Jan 2015 12:23:51 -0800 Subject: [PATCH] Add share driver for HDS NAS Scale-out Platform Add share driver for HDS NAS Scale-out Platform via support for version 1.0 of SOPAPI rest interface. At this time NFSv3 is supported protocol. Basic supported operations are: create/delete share allow/deny IP based access control Change-Id: I2ab1bf832cfdeb1d3b81f1bd800586a902231ba5 Implements: blueprint hitachi-sop-manila-driver --- manila/exception.py | 4 + manila/opts.py | 2 + manila/share/drivers/hds/__init__.py | 0 manila/share/drivers/hds/sop.py | 407 ++++++++++++++ manila/tests/share/drivers/hds/__init__.py | 0 manila/tests/share/drivers/hds/test_sop.py | 617 +++++++++++++++++++++ requirements.txt | 1 + 7 files changed, 1031 insertions(+) create mode 100644 manila/share/drivers/hds/__init__.py create mode 100644 manila/share/drivers/hds/sop.py create mode 100644 manila/tests/share/drivers/hds/__init__.py create mode 100644 manila/tests/share/drivers/hds/test_sop.py diff --git a/manila/exception.py b/manila/exception.py index eeeff6aca4..caae700646 100644 --- a/manila/exception.py +++ b/manila/exception.py @@ -481,3 +481,7 @@ class InvalidSqliteDB(Invalid): class SSHException(ManilaException): message = _("Exception in SSH protocol negotiation or logic.") + + +class SopAPIError(Invalid): + message = _("%(err)s") diff --git a/manila/opts.py b/manila/opts.py index eebbdf589e..83fb5a4322 100644 --- a/manila/opts.py +++ b/manila/opts.py @@ -52,6 +52,7 @@ import manila.share.drivers.emc.driver import manila.share.drivers.generic import manila.share.drivers.glusterfs import manila.share.drivers.glusterfs_native +import manila.share.drivers.hds.sop import manila.share.drivers.huawei.huawei_nas import manila.share.drivers.ibm.gpfs import manila.share.drivers.netapp.cluster_mode @@ -106,6 +107,7 @@ _global_opt_lists = [ manila.share.drivers.generic.share_opts, manila.share.drivers.glusterfs.GlusterfsManilaShare_opts, manila.share.drivers.glusterfs_native.glusterfs_native_manila_share_opts, + manila.share.drivers.hds.sop.hdssop_share_opts, manila.share.drivers.huawei.huawei_nas.huawei_opts, manila.share.drivers.ibm.gpfs.gpfs_share_opts, manila.share.drivers.netapp.cluster_mode.NETAPP_NAS_OPTS, diff --git a/manila/share/drivers/hds/__init__.py b/manila/share/drivers/hds/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/share/drivers/hds/sop.py b/manila/share/drivers/hds/sop.py new file mode 100644 index 0000000000..12832b0554 --- /dev/null +++ b/manila/share/drivers/hds/sop.py @@ -0,0 +1,407 @@ +# Copyright (c) 2015 Hitachi Data Systems. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Hitachi Data Systems Scale-out-Platform Manila Driver. +""" + +import base64 +import socket +import time + +import httplib2 +from oslo_config import cfg +from oslo_serialization import jsonutils as json +from oslo_utils import units +import six + +from manila import exception +from manila.i18n import _LW +from manila.openstack.common import log as logging +from manila.share import driver + + +LOG = logging.getLogger(__name__) + +hdssop_share_opts = [ + cfg.StrOpt('hdssop_target', + help='Specifies the SOPAPI cluster VIP. ' + 'It is of the form https://.'), + cfg.StrOpt('hdssop_adminuser', + help='Specifies the sop admin user'), + cfg.StrOpt('hdssop_adminpassword', + help='Specifies the sop admin user password', + secret=True) +] + +CONF = cfg.CONF +CONF.register_opts(hdssop_share_opts) + + +class SopShareDriver(driver.ShareDriver): + """Execute commands relating to Shares.""" + + def __init__(self, db, *args, **kwargs): + super(SopShareDriver, self).__init__(False, *args, **kwargs) + self.db = db + self.configuration.append_config_values(hdssop_share_opts) + self.backend_name = self.configuration.safe_get( + 'share_backend_name') or 'HDS_SOP' + self.sop_target = self.configuration.safe_get('hdssop_target') + self.sopuser = self.configuration.safe_get('hdssop_adminuser') + self.soppassword = self.configuration.safe_get('hdssop_adminpassword') + + def get_sop_auth_header(self): + return 'Basic ' + base64.b64encode( + self.sopuser + ':' + + self.soppassword).encode('utf-8').decode('ascii') + + def _wait_for_job_completion(self, httpclient, job_uri): + """Wait for job identified by job_uri to complete.""" + count = 0 + headers = dict(Authorization=self.get_sop_auth_header()) + + # NOTE(jasonsb): timeout logic here needs be revisited after + # load testing results are in. + while True: + if count > 300: + raise exception.SopAPIError(err=_('job timed out')) + + resp_headers, resp_content = httpclient.request(job_uri, 'GET', + body='', + headers=headers) + if int(resp_headers['status']) != 200: + raise exception.SopAPIError(err=_('error getting job status')) + + job = json.loads(resp_content) + if job['properties']['completion-status'] == 'ERROR': + raise exception.SopAPIError(err=_('job errored out')) + if job['properties']['completion-status'] == 'COMPLETE': + return job + time.sleep(1) + count += 1 + + def _add_file_system_sopapi(self, httpclient, payload): + """Add a new filesystem via SOPAPI.""" + sopuri = '/file-systems/' + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi' + sopuri + payload_json = json.dumps(payload) + resp_headers, resp_content = httpclient.request(uri, 'POST', + body=payload_json, + headers=headers) + resp_code = int(resp_headers['status']) + if resp_code == 202: + job_loc = resp_headers['location'] + self._wait_for_job_completion(httpclient, job_loc) + else: + raise exception.SopAPIError( + err=(_('received error: %s') % + resp_content['messages'][0]['message'])) + + def _add_share_sopapi(self, httpclient, payload): + """Add a new filesystem via SOPAPI.""" + sopuri = '/shares/' + headers = dict(Authorization=self.get_sop_auth_header()) + payload_json = json.dumps(payload) + uri = self.sop_target + '/sopapi' + sopuri + resp_headers, resp_content = httpclient.request(uri, 'POST', + body=payload_json, + headers=headers) + resp_code = int(resp_headers['status']) + if resp_code == 202: + job_loc = resp_headers['location'] + job = self._wait_for_job_completion(httpclient, job_loc) + if job['properties']['completion-status'] == 'COMPLETE': + return job['properties']['resource-name'] + else: + raise exception.SopAPIError(err=_('received error: %s') % + resp_headers['status']) + + def _get_file_system_id_by_name(self, httpclient, fsname): + + sopuri = '/file-systems/list?name=' + fsname + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi' + sopuri + resp_headers, resp_content = httpclient.request(uri, 'GET', + body='', + headers=headers) + + response = json.loads(resp_content) + num_of_resources = 0 + if int(resp_headers['status']) != 200 and 'messages' in response: + raise exception.SopAPIError( + err=(_('received error: %s') % + response['messages'][0]['message'])) + resource_list = [] + resource_list = response['list'] + num_of_resources = len(resource_list) + if num_of_resources <= 0: + return '' + return resource_list[0]['id'] + + def _get_share_id_by_name(self, httpclient, share_name): + """Look up share given the share name.""" + sopuri = '/shares/list?name=' + share_name + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi' + sopuri + resp_headers, resp_content = httpclient.request(uri, 'GET', + body='', + headers=headers) + response = json.loads(resp_content) + num_of_resources = 0 + if int(resp_headers['status']) != 200 and 'messages' in response: + raise exception.SopAPIError( + err=(_('received error: %s') % + response['messages'][0]['message'])) + resource_list = response['list'] + num_of_resources = len(resource_list) + if num_of_resources == 0: + return '' + return resource_list[0]['id'] + + def create_share(self, ctx, share, share_server=None): + """Create new share on HDS Scale-out Platform.""" + sharesize = int(six.text_type(share['size'])) + + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + + if share['share_proto'] != 'NFS': + raise exception.InvalidShare( + reason=(_('Invalid NAS protocol supplied: %s.') % + share['share_proto'])) + + payload = { + 'quota': sharesize * units.Gi, + 'enabled': True, + 'description': '', + 'record-access-time': True, + 'tags': '', + 'space-hwm': 90, + 'space-lwm': 70, + 'name': share['id'], + } + self._add_file_system_sopapi(httpclient, payload) + payload = { + 'description': '', + 'type': 'NFS', + 'enabled': True, + 'tags': '', + 'name': share['id'], + 'file-system-id': self._get_file_system_id_by_name( + httpclient, share['id']), + } + return self.sop_target + ':/' + self._add_share_sopapi( + httpclient, payload) + + def _delete_file_system_sopapi(self, httpclient, fs_id): + """Delete filesystem on SOP.""" + sopuri = '/file-systems/' + fs_id + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi' + sopuri + resp_headers, resp_content = httpclient.request(uri, 'DELETE', + body='', + headers=headers) + resp_code = int(resp_headers['status']) + if resp_code == 202: + job_loc = resp_headers['location'] + self._wait_for_job_completion(httpclient, job_loc) + else: + raise exception.SopAPIError(err=_('received error: %s') % + resp_headers['status']) + + def _delete_share_sopapi(self, httpclient, share_id): + """Delete share on SOP.""" + sopuri = '/shares/' + share_id + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi' + sopuri + resp_headers, resp_content = httpclient.request(uri, 'DELETE', + body='', + headers=headers) + resp_code = int(resp_headers['status']) + if resp_code == 202: + job_loc = resp_headers['location'] + self._wait_for_job_completion(httpclient, job_loc) + else: + raise exception.SopAPIError(err=_('received error: %s') % + resp_headers['status']) + + def delete_share(self, context, share, share_server=None): + """Remove a share from Sop volume.""" + + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + self._delete_share_sopapi( + httpclient, + self._get_share_id_by_name(httpclient, share['id'])) + self._delete_file_system_sopapi( + httpclient, + self._get_file_system_id_by_name(httpclient, share['id'])) + + def create_snapshot(self, context, snapshot, share_server=None): + """Not currently supported on HDS Scale-out Platform.""" + raise NotImplementedError() + + def create_share_from_snapshot(self, context, share, snapshot, + share_server=None): + """Not currently supported on HDS Scale-out Platform.""" + raise NotImplementedError() + + def delete_snapshot(self, context, snapshot, share_server=None): + """Not currently supported on HDS Scale-out Platform.""" + raise NotImplementedError() + + def allow_access(self, context, share, access, share_server=None): + """Allow access to a share. + + Currently only IP based access control is supported. + """ + + if access['access_type'] != 'ip': + raise exception.InvalidShareAccess( + reason=_('only IP access type allowed')) + + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + sop_share_id = self._get_share_id_by_name(httpclient, share['id']) + + if access['access_level'] == 'rw': + access_level = True + elif access['access_level'] == 'ro': + access_level = False + else: + raise exception.InvalidShareAccess( + reason=(_('Unsupported level of access was provided - %s') % + access['access_level'])) + payload = { + 'action': 'add-access-rule', + 'all-squash': True, + 'anongid': 65534, + 'anonuid': 65534, + 'host-specification': access['access_to'], + 'description': '', + 'read-write': access_level, + 'root-squash': False, + 'tags': 'nfs', + 'name': '%s-%s' % (share['id'], access['access_to']), + } + sopuri = '/shares/' + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi' + sopuri + sop_share_id + resp_headers, resp_content = httpclient.request( + uri, 'POST', + body=json.dumps(payload), + headers=headers) + resp_code = int(resp_headers['status']) + if resp_code == 202: + job_loc = resp_headers['location'] + self._wait_for_job_completion(httpclient, job_loc) + else: + raise exception.SopAPIError(err=_('received error: %s') % + resp_headers['status']) + + def deny_access(self, context, share, access, share_server=None): + """Deny access to a share. + + Currently only IP based access control is supported. + """ + if access['access_type'] != 'ip': + LOG.warn(_LW('Only ip access type allowed.')) + return + + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + sop_share_id = self._get_share_id_by_name(httpclient, share['id']) + payload = { + 'action': 'delete-access-rule', + 'name': '%s-%s' % (share['id'], access['access_to']), + } + + sopuri = '/shares/' + sop_share_id + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi' + sopuri + resp_headers, resp_content = httpclient.request( + uri, 'POST', + body=json.dumps(payload), + headers=headers) + resp_code = int(resp_headers['status']) + if resp_code == 202: + job_loc = resp_headers['location'] + self._wait_for_job_completion(httpclient, job_loc) + else: + raise exception.SopAPIError(err=_('received error: %s') % + resp_headers['status']) + + def check_for_setup_error(self): + """Check for setup error. + + Socket timeout set for 5 seconds to verify SOPAPI rest + interface is reachable and the credentials will allow us + to login. + """ + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi/clusters' + try: + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=5) + resp_headers, resp_content = httpclient.request(uri, 'GET', + body='', + headers=headers) + response = json.loads(resp_content) + if 'messages' in response: + soperror = _('received error: %(code)s: %(msg)s') % { + 'code': response['messages'][0]['code'], + 'msg': response['messages'][0]['message'], + } + raise exception.SopAPIError(err=soperror) + except socket.timeout: + raise exception.SopAPIError( + err=_('connection to SOPAPI timed out')) + + def _get_sop_filesystem_stats(self): + """Calculate cluster storage capacity and return in GiB.""" + headers = dict(Authorization=self.get_sop_auth_header()) + uri = self.sop_target + '/sopapi/clusters' + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + resp_headers, resp_content = httpclient.request(uri, 'GET', + body='', + headers=headers) + response = json.loads(resp_content) + if resp_content is not None: + for cluster in response['element-links']: + (resp_headers, resp_content) = httpclient.request( + cluster, + 'GET', + body='', + headers=headers) + response = json.loads(resp_content) + totalspace = int(response['properties'] + ['total-storage-capacity']) / units.Gi + spaceavail = int(response['properties'] + ['total-storage-available']) / units.Gi + return (totalspace, spaceavail) + + def _update_share_stats(self): + """Retrieve stats info from SOPAPI.""" + totalspace, spaceavail = self._get_sop_filesystem_stats() + data = dict( + share_backend_name=self.backend_name, + vendor_name='Hitach Data Systems', + storage_protocol='NFS', + total_capacity_gb=totalspace, + free_capacity_gb=spaceavail) + super(SopShareDriver, self)._update_share_stats(data) diff --git a/manila/tests/share/drivers/hds/__init__.py b/manila/tests/share/drivers/hds/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/tests/share/drivers/hds/test_sop.py b/manila/tests/share/drivers/hds/test_sop.py new file mode 100644 index 0000000000..330277d152 --- /dev/null +++ b/manila/tests/share/drivers/hds/test_sop.py @@ -0,0 +1,617 @@ +# Copyright (c) 2015 Hitachi Data Systems. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Unit tests for the Hitachi Data Systems Scale-out Platform manila driver.""" + +import time + +import httplib2 +import mock +from oslo_config import cfg +from oslo_serialization import jsonutils as json +from oslo_utils import units + +from manila import context +from manila import exception +from manila.share import configuration as config +from manila.share.drivers.hds import sop +from manila import test +from manila.tests import fake_share + + +CONF = cfg.CONF + +fake_authorization = {'Authorization': u'Basic ZmFrZXVzZXI6ZmFrZXBhc3N3b3Jk'} + + +class SopShareDriverTestCase(test.TestCase): + """Tests SopShareDriver.""" + + def setUp(self): + super(SopShareDriverTestCase, self).setUp() + self._context = context.get_admin_context() + self.server = { + 'instance_id': 'fake_instance_id', + 'ip': 'fake_ip', + 'username': 'fake_username', + 'password': 'fake_password', + 'pk_path': 'fake_pk_path', + 'backend_details': { + 'ip': '1.2.3.4', + 'instance_id': 'fake', + }, + } + CONF.set_default('hdssop_target', 'https://1.2.3.4') + CONF.set_default('hdssop_adminuser', 'fakeuser') + CONF.set_default('hdssop_adminpassword', 'fakepassword') + CONF.set_default('driver_handles_share_servers', False) + + self.fake_conf = config.Configuration(None) + self._db = mock.Mock() + self._driver = sop.SopShareDriver( + self._db, configuration=self.fake_conf) + self.share = fake_share.fake_share(share_proto='NFS') + self._driver.share_backend_name = 'HDS_SOP' + + def test_add_file_system_sopapi(self): + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + + httpretval = ({'status': '202', + 'content-length': '0', + 'x-sopapi-version': '1.0.0', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'location': 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'date': 'Tue, 20 Jan 2015 22:41:29 GMT'}, '') + + self.mock_object(httpclient, 'request', + mock.Mock(return_value=httpretval)) + self.mock_object(self._driver, '_wait_for_job_completion', mock.Mock()) + + fakepayload1 = { + 'quota': 145 * units.Gi, + 'enabled': True, + 'description': '', + 'record-access-time': True, + 'tags': '', + 'space-hwm': 90, + 'space-lwm': 70, + 'name': 'fakeid', + } + + fsadd = self._driver._add_file_system_sopapi(httpclient, fakepayload1) + self.assertEqual(None, fsadd) + httpclient.request.assert_called_once_with( + 'https://' + + self.server['backend_details']['ip'] + + '/sopapi/file-systems/', + 'POST', + body=json.dumps(fakepayload1), + headers=fake_authorization) + self._driver._wait_for_job_completion.assert_called_once_with( + httpclient, + 'https://1.2.3.4/sopapi/jobs/fakeuuid') + + def test_add_file_system_sopapi_belowminsize(self): + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + + httpretval = ({'status': '400', + 'content-type': 'application/jsson', + 'transfer-encoding': 'chunked', + 'x-sopapi-version': '1.0.0', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'location': 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'date': 'Tue, 20 Jan 2015 22:41:29 GMT'}, + {'messages': [{'category': 1, + 'message': '''"Property 'quota' is inv''' + 'alid. Specify a value from 137438953472 ' + 'to 6755399441055744."', + 'code': 'schema_number_min_constraint', + 'type': 'error'}, + ]}) + self.mock_object(httpclient, 'request', + mock.Mock(return_value=httpretval)) + self.mock_object(self._driver, '_wait_for_job_completion', mock.Mock()) + + fakepayload = { + 'quota': 3 * units.Gi, + 'enabled': True, + 'description': '', + 'record-access-time': True, + 'tags': '', + 'space-hwm': 90, + 'space-lwm': 70, + 'name': 'fakeid', + } + self.assertRaises(exception.SopAPIError, + self._driver._add_file_system_sopapi, + httpclient, fakepayload) + httpclient.request.assert_called_once_with( + 'https://' + + self.server['backend_details']['ip'] + + '/sopapi/file-systems/', + 'POST', + body=json.dumps(fakepayload), + headers=fake_authorization) + self.assertEqual(False, self._driver._wait_for_job_completion.called) + + def test_wait_for_job_completion_simple(self): + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + + httpreturn = [ + ({'status': '200', + 'content-location': 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'transfer-encoding': 'chunked', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'x-sopapi-version': '1.0.0', + 'date': 'Wed, 21 Jan 2015 04:49:51 GMT', + 'content-type': 'application/json'}, + '{"id":"fakeuuid","properties":{"' + 'resource-name":"","resource-type":"share","creation-timestam' + 'p":1421815791,"completion-status":"PROCESSING","completion-d' + 'etails":"Saving changes","completion-substatus":"RUNNING","r' + 'esource-action":"ADD","percent-complete":75,"resource-id":"b' + 'fakeuuid","target-node-name":"Node005","target-node-id":"fak' + 'euuid","spawned-jobs":false,"spawned-jobs-list-uri":""}}'), + ({'status': '200', + 'content-location': 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'transfer-encoding': 'chunked', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'x-sopapi-version': '1.0.0', + 'date': 'Wed, 21 Jan 2015 04:49:51 GMT', + 'content-type': 'application/json'}, + '{"id":"fakeuuid","properties":{"' + 'resource-name":"fakeuuid","resou' + 'rce-type":"share","creation-timestamp":1421815791,"completio' + 'n-status":"COMPLETE","completion-details":"Adding share comp' + 'leted","completion-substatus":"OK","resource-action":"ADD","' + 'percent-complete":100,"resource-id":"fakeuuid' + '","target-node-name":"Node005","target-node-id"' + ':"fakeuuid","spawned-jobs":false' + ',"spawned-jobs-list-uri":""}}'), + ] + + self.mock_object(httpclient, 'request', + mock.Mock(side_effect=httpreturn)) + + fsadd = self._driver._wait_for_job_completion(httpclient, 'fakeuri') + + expectedresult = { + u'id': u'fakeuuid', + u'properties': { + u'completion-details': + u'Adding share completed', + u'completion-status': u'COMPLETE', + u'completion-substatus': u'OK', + u'creation-timestamp': 1421815791, + u'percent-complete': 100, + u'resource-action': u'ADD', + u'resource-id': u'fakeuuid', + u'resource-name': u'fakeuuid', + u'resource-type': u'share', + u'spawned-jobs': False, + u'spawned-jobs-list-uri': u'', + u'target-node-id': u'fakeuuid', + u'target-node-name': u'Node005', + }, + } + self.assertEqual(expectedresult, fsadd) + httpcalls = [mock.call('fakeuri', + 'GET', + body='', + headers=fake_authorization) for x in xrange(2)] + self.assertEqual(httpcalls, httpclient.request.call_args_list) + + def test_wait_for_job_completion_notimeout(self): + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + + httpreturn = [({'status': '200', + 'content-location': + 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'transfer-encoding': 'chunked', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'x-sopapi-version': '1.0.0', + 'date': 'Wed, 21 Jan 2015 04:49:51 GMT', + 'content-type': 'application/json'}, + '{"id":"fakeuuid","properties":{"resource-name":"","re' + 'source-type":"share","creation-timestamp":1421815791,' + '"completion-status":"PROCESSING","completion-details"' + ':"Saving changes","completion-substatus":"RUNNING","r' + 'esource-action":"ADD","percent-complete":75,"resource' + '-id":"fakeuuid","target-node-name":"Node005","target-' + 'node-id":"fakeuuid","spawned-jobs":false,"spawned-job' + 's-list-uri":""}}') for x in xrange(200) + ] + + httpreturn.append(({'status': '200', + 'content-location': + 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'transfer-encoding': 'chunked', + 'set-cookie': + 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'x-sopapi-version': '1.0.0', + 'date': 'Wed, 21 Jan 2015 04:49:51 GMT', + 'content-type': 'application/json'}, + '{"id":"fakeuuid","properties":{"resource-name":"fa' + 'keuuid","resource-type":"share","creation-timestam' + 'p":1421816291,"completion-status":"COMPLETE","comp' + 'letion-details":"Adding share completed","completi' + 'on-substatus":"OK","resource-action":"ADD","percen' + 't-complete":100,"resource-id":"fakeuuid","target-n' + 'ode-name":"Node005","target-node-id":"fakeuuid","s' + 'pawned-jobs":false,"spawned-jobs-list-uri":""}}')) + + self.mock_object(httpclient, 'request', + mock.Mock(side_effect=httpreturn)) + self.mock_object(time, 'sleep', mock.Mock()) + + fsadd = self._driver._wait_for_job_completion(httpclient, 'fakeuri') + + expectedresult = { + u'id': u'fakeuuid', + u'properties': { + u'completion-details': + u'Adding share completed', + u'completion-status': u'COMPLETE', + u'completion-substatus': u'OK', + u'creation-timestamp': 1421816291, + u'percent-complete': 100, + u'resource-action': u'ADD', + u'resource-id': u'fakeuuid', + u'resource-name': u'fakeuuid', + u'resource-type': u'share', + u'spawned-jobs': False, + u'spawned-jobs-list-uri': u'', + u'target-node-id': u'fakeuuid', + u'target-node-name': u'Node005', + }, + } + self.assertEqual(expectedresult, fsadd) + httpcalls = [mock.call('fakeuri', + 'GET', + body='', + headers=fake_authorization) + for x in xrange(201)] + self.assertEqual(httpcalls, httpclient.request.call_args_list) + timecalls = [mock.call(1) for x in xrange(200)] + self.assertEqual(timecalls, time.sleep.call_args_list) + + def test_wait_for_job_completion_timeout(self): + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + + httpret = [({'status': '200', + 'content-location': 'https://1.2.3.4/sopapi/jobs/' + 'fakeuuid', + 'transfer-encoding': 'chunked', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'x-sopapi-version': '1.0.0', + 'date': 'Wed, 21 Jan 2015 04:49:51 GMT', + 'content-type': 'application/json'}, + '{"id":"fakeuuid","properties' + '":{"resource-name":"","resource-type":"share","creation-' + 'timestamp":1421815791,"completion-status":"PROCESSING","' + 'completion-details":"Saving changes","completion-substat' + 'us":"RUNNING","resource-action":"ADD","percent-complete"' + ':75,"resource-id":"fakeuuid"' + ',"target-node-name":"Node005","target-node-id":"fakeuuid' + '","spawned-jobs":false,"spawned-jobs-list-uri":""}}') + for x in xrange(301)] + + httpret.append(({'status': '200', + 'content-location': + 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'transfer-encoding': 'chunked', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'x-sopapi-version': '1.0.0', + 'date': 'Wed, 21 Jan 2015 04:49:51 GMT', + 'content-type': 'application/json'}, + '{"id":"fakeuuid","propert' + 'ies":{"resource-name":"fakeuuid","resource-type":"sha' + 're","creation-timestamp":1421815791,"completion-statu' + 's":"COMPLETE","completion-details":"Adding share comp' + 'leted","completion-substatus":"OK","resource-action"' + ':"ADD","percent-complete": 100,"resource-id":"fakeuui' + 'd","target-node-name":"Node005","target-node-id":"fa' + 'keuuid","spawned-jobs":false,"spawned-jobs-list-uri"' + ':""}}')) + + self.mock_object(httpclient, 'request', mock.Mock(side_effect=httpret)) + self.mock_object(time, 'sleep', mock.Mock()) + + self.assertRaises(exception.SopAPIError, + self._driver._wait_for_job_completion, + httpclient, 'fakeuri') + httpcalls = [mock.call('fakeuri', + 'GET', + body='', + headers=fake_authorization) + for x in xrange(301)] + self.assertEqual(httpcalls, httpclient.request.call_args_list) + timecalls = [mock.call(1) for x in xrange(301)] + self.assertEqual(timecalls, time.sleep.call_args_list) + + def test_add_share_sopapi(self): + httpclient = httplib2.Http(disable_ssl_certificate_validation=True, + timeout=None) + + httpret = ({'status': '202', + 'content-length': '0', + 'x-sopapi-version': '1.0.0', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;Secure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'location': 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'date': 'Wed, 21 Jan 2015 05:29:35 GMT'}, '') + self.mock_object(httpclient, 'request', + mock.Mock(return_value=httpret)) + + waitforret = json.loads('{"id":"fakeuuid' + '","properties":{"resource-name":"fakeuuid' + '","resource-type":"share",' + '"creation-timestamp":1421815791,"c' + 'ompletion-status":"COMPLETE","completion-de' + 'tails":"Adding share completed","completion' + '-substatus":"OK","resource-action":"ADD","p' + 'ercent-complete":100,"resource-id":"fakeuui' + 'd","target-node-name":"Node005","target-nod' + 'e-id":"fakeuuid1","spawned-jobs":false,"spaw' + 'ned-jobs-list-uri":""}}') + self.mock_object(self._driver, '_wait_for_job_completion', + mock.Mock(return_value=waitforret)) + + fakepayload = { + 'description': '', + 'type': 'NFS', + 'enabled': True, + 'tags': '', + 'name': 'fakeuuid', + 'file-system-id': 'fakeuuid', + } + fsadd = self._driver._add_share_sopapi(httpclient, fakepayload) + self.assertEqual('fakeuuid', fsadd) + httpcalls = [mock.call('https://' + + self.server['backend_details']['ip'] + + '/sopapi/shares/', + 'POST', + body=json.dumps(fakepayload), + headers=fake_authorization)] + self.assertEqual(httpcalls, httpclient.request.call_args_list) + self._driver._wait_for_job_completion.assert_called_once_with( + httpclient, 'https://' + + self.server['backend_details']['ip'] + + '/sopapi/jobs/fakeuuid') + + def test_create_share_success(self): + + self.mock_object(self._driver, '_add_file_system_sopapi', mock.Mock()) + self.mock_object(self._driver, '_get_file_system_id_by_name', + mock.Mock(return_value='fakeuuid')) + self.mock_object(self._driver, '_add_share_sopapi', + mock.Mock(return_value='fakeuuid')) + + result = self._driver.create_share( + self._context, self.share, share_server=self.server) + + self.assertEqual('https://1.2.3.4:/fakeuuid', result) + + fakepayload = { + 'quota': 1073741824, + 'enabled': True, + 'description': '', + 'record-access-time': True, + 'tags': '', + 'space-hwm': 90, + 'space-lwm': 70, + 'name': 'fakeid', + } + + fakepayload1 = { + 'description': '', + 'type': 'NFS', + 'enabled': True, + 'tags': '', + 'name': 'fakeid', + 'file-system-id': 'fakeuuid', + } + self._driver._add_file_system_sopapi.assert_called_once_with( + mock.ANY, fakepayload) + self._driver._get_file_system_id_by_name.assert_called_once_with( + mock.ANY, 'fakeid') + self._driver._add_share_sopapi.assert_called_once_with( + mock.ANY, fakepayload1) + + def test_get_share_stats_refresh_false(self): + self._driver._stats = {'fake_key': 'fake_value'} + + result = self._driver.get_share_stats(False) + self.assertEqual(result, self._driver._stats) + + def test_get_share_stats_refresh_true(self): + test_data = { + 'driver_handles_share_servers': False, + 'share_backend_name': 'HDS_SOP', + 'vendor_name': 'Hitach Data Systems', + 'driver_version': '1.0', + 'storage_protocol': 'NFS', + 'reserved_percentage': 0, + 'QoS_support': False, + 'total_capacity_gb': 1234, + 'free_capacity_gb': 2345, + } + self.mock_object(self._driver, '_get_sop_filesystem_stats', + mock.Mock(return_value=(1234, 2345))) + self._driver._update_share_stats() + self.assertEqual(test_data, self._driver._stats) + self._driver._get_sop_filesystem_stats.assert_called_once_with() + + def test_allow_access_rw(self): + payload = { + 'action': 'add-access-rule', + 'all-squash': True, + 'anongid': 65534, + 'anonuid': 65534, + 'host-specification': '1.2.3.4', + 'description': '', + 'read-write': True, + 'root-squash': False, + 'tags': 'nfs', + 'name': 'fakeid-1.2.3.4' + } + + self.mock_object(self._driver, '_get_share_id_by_name', + mock.Mock(return_value='fakeuuid')) + self.mock_object(self._driver, '_wait_for_job_completion', mock.Mock()) + self.mock_object(httplib2.Http, 'request', mock.Mock( + return_value=({'status': '202', + 'content-length': '0', + 'x-sopapi-version': '1.0.0', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;S' + 'ecure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'location': 'https://1.2.3.4/sopapi/jobs/fakeuu' + 'id', + 'date': 'Wed, 21 Jan 2015 05:29:35 GMT'}, ''))) + + access = { + 'access_type': 'ip', + 'access_to': '1.2.3.4', + 'access_level': 'rw', + } + self._driver.allow_access( + self._context, self.share, access, share_server=self.server) + + headers = dict(Authorization=self._driver.get_sop_auth_header()) + + httplib2.Http.request.assert_called_once_with( + 'https://1.2.3.4/sopapi/shares/fakeuuid', 'POST', + body=json.dumps(payload), + headers=headers) + self._driver._get_share_id_by_name.assert_called_once_with( + mock.ANY, 'fakeid') + self._driver._wait_for_job_completion.assert_called_once_with( + mock.ANY, 'https://' + + self.server['backend_details']['ip'] + + '/sopapi/jobs/fakeuuid') + + def test_allow_access_ro(self): + payload = { + 'action': 'add-access-rule', + 'all-squash': True, + 'anongid': 65534, + 'anonuid': 65534, + 'host-specification': '1.2.3.4', + 'description': '', + 'read-write': False, + 'root-squash': False, + 'tags': 'nfs', + 'name': 'fakeid-1.2.3.4' + } + + self.mock_object(self._driver, '_get_share_id_by_name', + mock.Mock(return_value='fakeuuid')) + self.mock_object(self._driver, '_wait_for_job_completion', mock.Mock()) + self.mock_object(httplib2.Http, 'request', mock.Mock( + return_value=({'status': '202', + 'content-length': '0', + 'x-sopapi-version': '1.0.0', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;S' + 'ecure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'location': 'https://1.2.3.4/sopapi/jobs/fakeuu' + 'id', + 'date': 'Wed, 21 Jan 2015 05:29:35 GMT'}, ''))) + + access = { + 'access_type': 'ip', + 'access_to': '1.2.3.4', + 'access_level': 'ro', + } + self._driver.allow_access( + self._context, self.share, access, share_server=self.server) + + headers = dict(Authorization=self._driver.get_sop_auth_header()) + + httplib2.Http.request.assert_called_once_with( + 'https://1.2.3.4/sopapi/shares/fakeuuid', 'POST', + body=json.dumps(payload), + headers=headers) + self._driver._get_share_id_by_name.assert_called_once_with( + mock.ANY, 'fakeid') + self._driver._wait_for_job_completion.assert_called_once_with( + mock.ANY, 'https://' + + self.server['backend_details']['ip'] + + '/sopapi/jobs/fakeuuid') + + def test_deny_access(self): + payload = { + 'action': 'delete-access-rule', + 'name': 'fakeid-1.2.3.4', + } + + self.mock_object(self._driver, '_get_share_id_by_name', + mock.Mock(return_value='fakeuuid')) + self.mock_object(self._driver, '_wait_for_job_completion', mock.Mock()) + self.mock_object(httplib2.Http, 'request', mock.Mock( + return_value=({'status': '202', 'content-length': '0', + 'x-sopapi-version': '1.0.0', + 'set-cookie': 'JSESSIONID=abcdef;Path=/sopapi;S' + 'ecure', + 'expires': 'Thu, 01 Jan 1970 00:00:00 GMT', + 'server': 'Jetty(8.1.3.v20120416)', + 'location': 'https://1.2.3.4/sopapi/jobs/fakeuuid', + 'date': 'Wed, 21 Jan 2015 05:29:35 GMT'}, ''))) + + access = { + 'access_type': 'ip', + 'access_to': '1.2.3.4', + 'access_level': 'rw', + } + self._driver.deny_access( + self._context, self.share, access, share_server=self.server) + + headers = dict(Authorization=self._driver.get_sop_auth_header()) + + httplib2.Http.request.assert_called_once_with( + 'https://1.2.3.4/sopapi/shares/fakeuuid', 'POST', + body=json.dumps(payload), + headers=headers) + self._driver._get_share_id_by_name.assert_called_once_with( + mock.ANY, 'fakeid') + self._driver._wait_for_job_completion.assert_called_once_with( + mock.ANY, 'https://' + + self.server['backend_details']['ip'] + + '/sopapi/jobs/fakeuuid') diff --git a/requirements.txt b/requirements.txt index f503d18fb7..4b34020db9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ alembic>=0.7.2 Babel>=1.3 eventlet>=0.16.1 greenlet>=0.3.2 +httplib2>=0.7.5 iso8601>=0.1.9 lxml>=2.3 oslo.config>=1.6.0 # Apache-2.0