From 54d4769a3bcdefd18139b8fd59430a2a80e18a9d Mon Sep 17 00:00:00 2001 From: AmitKumarDas Date: Wed, 25 Jun 2014 17:38:52 +0530 Subject: [PATCH] Adds cinder iscsi driver for CloudByte storage This patch satisfies the basic requirements for a cinder driver. It has the necessary code to provision storage volume, snapshot and clone. Driver certification results url https://bugs.launchpad.net/cinder/+bug/1380126 Change-Id: Icd91eff614fa6b3e61e48edccda4bd7bf3955b60 Implements: blueprint CloudByte-ElastiStor-Cinder-Driver --- cinder/tests/test_cloudbyte.py | 1198 ++++++++++++++++++ cinder/volume/drivers/cloudbyte/__init__.py | 0 cinder/volume/drivers/cloudbyte/cloudbyte.py | 951 ++++++++++++++ cinder/volume/drivers/cloudbyte/options.py | 74 ++ 4 files changed, 2223 insertions(+) create mode 100644 cinder/tests/test_cloudbyte.py create mode 100644 cinder/volume/drivers/cloudbyte/__init__.py create mode 100644 cinder/volume/drivers/cloudbyte/cloudbyte.py create mode 100644 cinder/volume/drivers/cloudbyte/options.py diff --git a/cinder/tests/test_cloudbyte.py b/cinder/tests/test_cloudbyte.py new file mode 100644 index 000000000..1e7855fe5 --- /dev/null +++ b/cinder/tests/test_cloudbyte.py @@ -0,0 +1,1198 @@ +# Copyright 2015 CloudByte Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" Test class for cloudbyte's cinder driver. + +This involves mocking of elasticenter's json responses +when a method of this driver is unit tested. +""" + +import json + +import mock +import testtools +from testtools import ExpectedException +from testtools.matchers import Contains + +from cinder import exception +from cinder.volume import configuration as conf +from cinder.volume.drivers.cloudbyte.cloudbyte import CloudByteISCSIDriver + +# A fake list account response of cloudbyte's elasticenter +FAKE_LIST_ACCOUNT_RESPONSE = """{ "listAccountResponse" : { + "count":1 , + "account" : [{ + "id": "d13a4e9e-0c05-4d2d-8a5e-5efd3ef058e0", + "name": "CustomerA", + "simpleid": 1, + "description": "None", + "iqnname": "iqn.2014-05.cvsacc1", + "availIOPS": 508, + "totaliops": 2000, + "usedIOPS": 1492, + "volumes": [], + "storageBuckets": [], + "tsms": [], + "qosgroups": [], + "filesystemslist": [], + "currentUsedSpace": 53179, + "currentAvailableSpace": 1249349, + "currentThroughput": 156, + "currentIOPS": 33, + "currentLatency": 0, + "currentThrottle": 0, + "numericquota": 3145728.0, + "currentnumericquota": 1253376.0, + "currentavailablequota": 1892352.0, + "revisionnumber": 1 + }] + }}""" + +# A fake list tsm response of cloudbyte's elasticenter +FAKE_LIST_TSM_RESPONSE = """{ "listTsmResponse" : { + "count":1 , + "listTsm" : [{ + "id": "955eaf34-4221-3a77-82d0-99113b126fa8", + "simpleid": 2, + "name": "openstack", + "ipaddress": "172.16.50.40", + "accountname": "CustomerA", + "sitename": "BLR", + "clustername": "HAGrp1", + "controllerName": "Controller", + "controlleripaddress": "172.16.50.6", + "clusterstatus": "Online", + "hapoolstatus": "ONLINE", + "hapoolname": "pool", + "hapoolavailiops": 1700, + "hapoolgrace": true, + "hapoolavailtput": 6800, + "poollatency": 10, + "accountid": "d13a4e9e-0c05-4d2d-8a5e-5efd3ef058e0", + "controllerid": "8c2f7084-99c0-36e6-9cb7-205e3ba4c813", + "poolid": "adcbef8f-2193-3f2c-9bb1-fcaf977ae0fc", + "datasetid": "87a23025-f2b2-39e9-85ac-9cda15bfed1a", + "storageBuckets": [], + "currentUsedSpace": 16384, + "currentAvailableSpace": 188416, + "currentTotalSpace": 204800, + "currentThroughput": 12, + "tpcontrol": "true", + "currentIOPS": 0, + "iopscontrol": "true", + "gracecontrol": "false", + "currentLatency": 0, + "currentThrottle": 0, + "iops": "1000", + "availIOPS": "500", + "availThroughput": "2000", + "usedIOPS": "500", + "usedThroughput": "2000", + "throughput": "4000", + "latency": "15", + "graceallowed": true, + "numericquota": 1048576.0, + "currentnumericquota": 204800.0, + "availablequota": 843776.0, + "blocksize": "4", + "type": "1", + "iqnname": "iqn.2014-05.cvsacc1.openstack", + "interfaceName": "em0", + "revisionnumber": 0, + "status": "Online", + "subnet": "16", + "managedstate": "Available", + "configurationstate": "sync", + "offlinenodes": "", + "pooltakeover": "noTakeOver", + "totalprovisionquota": "536576", + "haNodeStatus": "Available", + "ispooltakeoveronpartialfailure": true, + "filesystemslist": [], + "volumes": [], + "qosgrouplist": [] + }] + }}""" + +# A fake add QOS group response of cloudbyte's elasticenter +FAKE_ADD_QOS_GROUP_RESPONSE = """{ "addqosgroupresponse" : { + "qosgroup" : { + "id": "d73662ac-6db8-3b2c-981a-012af4e2f7bd", + "name": "QoS_DS1acc1openstacktsm", + "tsmid": "8146146e-f67b-3942-8074-3074599207a4", + "controllerid": "f1603e87-d1e6-3dcb-a549-7a6e77f82d86", + "poolid": "73b567c0-e57d-37b5-b765-9d70725f59af", + "parentid": "81ebdcbb-f73b-3337-8f32-222820e6acb9", + "tsmName": "openstacktsm", + "offlinenodes": "", + "sitename": "site1", + "clustername": "HA1", + "controllerName": "node1", + "clusterstatus": "Online", + "currentThroughput": 0, + "currentIOPS": 0, + "currentLatency": 0, + "currentThrottle": 0, + "iopsvalue": "(0/100)", + "throughputvalue": "(0/400)", + "iops": "100", + "iopscontrol": "true", + "throughput": "400", + "tpcontrol": "true", + "blocksize": "4k", + "latency": "15", + "graceallowed": false, + "type": "1", + "revisionnumber": 0, + "managedstate": "Available", + "configurationstate": "init", + "standardproviops": 0, + "operatingblocksize": 0, + "operatingcachehit": 0, + "operatingiops": 0, + "standardoperatingiops": 0 + } + }}""" + +# A fake create volume response of cloudbyte's elasticenter +FAKE_CREATE_VOLUME_RESPONSE = """{ "createvolumeresponse" : { + "jobid": "f94e2257-9515-4a44-add0-4b16cb1bcf67" + }}""" + +# A fake query async job response of cloudbyte's elasticenter +FAKE_QUERY_ASYNC_JOB_RESULT_RESPONSE = """{ "queryasyncjobresultresponse" : { + "accountid": "e8aca633-7bce-4ab7-915a-6d8847248467", + "userid": "a83d1030-1b85-40f7-9479-f40e4dbdd5d5", + "cmd": "com.cloudbyte.api.commands.CreateVolumeCmd", + "msg": "5", + "jobstatus": 1, + "jobprocstatus": 0, + "jobresultcode": 0, + "jobresulttype": "object", + "jobresult": { + "storage": { + "id": "92cfd601-bc1f-3fa7-8322-c492099f3326", + "name": "DS1", + "simpleid": 20, + "compression": "off", + "sync": "always", + "noofcopies": 1, + "recordsize": "4k", + "deduplication": "off", + "quota": "10G", + "path": "devpool1/acc1openstacktsm/DS1", + "tsmid": "8146146e-f67b-3942-8074-3074599207a4", + "poolid": "73b567c0-e57d-37b5-b765-9d70725f59af", + "mountpoint": "acc1DS1", + "currentUsedSpace": 0, + "currentAvailableSpace": 0, + "currentTotalSpace": 0, + "currentThroughput": 0, + "currentIOPS": 0, + "currentLatency": 0, + "currentThrottle": 0, + "tsmName": "openstacktsm", + "hapoolname": "devpool1", + "revisionnumber": 0, + "blocklength": "512B", + "nfsenabled": false, + "cifsenabled": false, + "iscsienabled": true, + "fcenabled": false + } + }, + "created": "2014-06-16 15:49:49", + "jobid": "f94e2257-9515-4a44-add0-4b16cb1bcf67" + }}""" + +# A fake list filesystem response of cloudbyte's elasticenter +FAKE_LIST_FILE_SYSTEM_RESPONSE = """{ "listFilesystemResponse" : { + "count":1 , + "filesystem" : [{ + "id": "c93df32e-3a99-3491-8e10-cf318a7f9b7f", + "name": "c93df32e3a9934918e10cf318a7f9b7f", + "simpleid": 34, + "type": "filesystem", + "revisionnumber": 1, + "path": "/cvsacc1DS1", + "clusterid": "8b404f12-7975-4e4e-8549-7abeba397fc9", + "clusterstatus": "Online", + "Tsmid": "955eaf34-4221-3a77-82d0-99113b126fa8", + "tsmType": "1", + "accountid": "d13a4e9e-0c05-4d2d-8a5e-5efd3ef058e0", + "poolid": "adcbef8f-2193-3f2c-9bb1-fcaf977ae0fc", + "controllerid": "8c2f7084-99c0-36e6-9cb7-205e3ba4c813", + "groupid": "663923c9-084b-3778-b13d-72f23d046b8d", + "parentid": "08de7c14-62af-3992-8407-28f5f053e59b", + "compression": "off", + "sync": "always", + "noofcopies": 1, + "recordsize": "4k", + "deduplication": "off", + "quota": "1T", + "unicode": "off", + "casesensitivity": "sensitive", + "readonly": false, + "nfsenabled": true, + "cifsenabled": false, + "iscsienabled": false, + "fcenabled": false, + "currentUsedSpace": 19968, + "currentAvailableSpace": 1028608, + "currentTotalSpace": 1048576, + "currentThroughput": 0, + "currentIOPS": 0, + "currentLatency": 0, + "currentThrottle": 0, + "numericquota": 1048576.0, + "status": "Online", + "managedstate": "Available", + "configurationstate": "sync", + "tsmName": "cvstsm1", + "ipaddress": "172.16.50.35", + "sitename": "BLR", + "clustername": "HAGrp1", + "controllerName": "Controller", + "hapoolname": "pool", + "hapoolgrace": true, + "tsmgrace": true, + "tsmcontrolgrace": "false", + "accountname": "CustomerA", + "groupname": "QoS_DS1cvsacc1cvstsm1", + "iops": "500", + "blocksize": "4", + "throughput": "2000", + "latency": "15", + "graceallowed": false, + "offlinenodes": "", + "tpcontrol": "true", + "iopscontrol": "true", + "tsmAvailIops": "8", + "tsmAvailTput": "32", + "iqnname": "", + "mountpoint": "cvsacc1DS1", + "pooltakeover": "noTakeOver", + "volumeaccessible": "true", + "localschedulecount": 0 + }] + }}""" + +# A fake list storage snapshot response of cloudbyte's elasticenter +FAKE_LIST_STORAGE_SNAPSHOTS_RESPONSE = """{ "listDatasetSnapshotsResponse" : { + "count":1 , + "snapshot" : [{ + "name": "DS1Snap1", + "path": "devpool1/acc1openstacktsm/DS1@DS1Snap1", + "availMem": "-", + "usedMem": "0", + "refer": "26K", + "mountpoint": "-", + "timestamp": "Mon Jun 16 2014 14:41", + "clones": 0, + "pooltakeover": "noTakeOver", + "managedstate": "Available" + }] + }}""" + +# A fake delete storage snapshot response of cloudbyte's elasticenter +FAKE_DELETE_STORAGE_SNAPSHOT_RESPONSE = """{ "deleteSnapshotResponse" : { + "DeleteSnapshot" : { + "status": "success" + } + }}""" + +# A fake update volume iscsi service response of cloudbyte's elasticenter +FAKE_UPDATE_VOLUME_ISCSI_SERVICE_RESPONSE = ( + """{ "updatingvolumeiscsidetails" : { + "viscsioptions" : { + "id": "0426c04a-8fac-30e8-a8ad-ddab2f08013a", + "volume_id": "12371e7c-392b-34b9-ac43-073b3c85f1d1", + "ag_id": "4459248d-e9f1-3d2a-b7e8-b5d9ce587fc1", + "ig_id": "527bd65b-ebec-39ce-a5e9-9dd1106cc0fc", + "iqnname": "iqn.2014-06.acc1.openstacktsm:acc1DS1", + "authmethod": "None", + "status": true, + "usn": "12371e7c392b34b9ac43073b3c85f1d1", + "initialdigest": "Auto", + "queuedepth": "32", + "inqproduct": 0, + "inqrevision": 0, + "blocklength": "512B" + }} + }""") + +# A fake list iscsi initiator response of cloudbyte's elasticenter +FAKE_LIST_ISCSI_INITIATOR_RESPONSE = """{ "listInitiatorsResponse" : { + "count":2 , + "initiator" : [{ + "id": "527bd65b-ebec-39ce-a5e9-9dd1106cc0fc", + "accountid": "86c5251a-9044-4690-b924-0d97627aeb8c", + "name": "ALL", + "netmask": "ALL", + "initiatorgroup": "ALL" + },{ + "id": "203e0235-1d5a-3130-9204-98e3f642a564", + "accountid": "86c5251a-9044-4690-b924-0d97627aeb8c", + "name": "None", + "netmask": "None", + "initiatorgroup": "None" + }] + }}""" + +# A fake delete file system response of cloudbyte's elasticenter +FAKE_DELETE_FILE_SYSTEM_RESPONSE = """{ "deleteResponse" : { + "response" : [{ + "code": "0", + "description": "success" + }] + }}""" + +# A fake create storage snapshot response of cloudbyte's elasticenter +FAKE_CREATE_STORAGE_SNAPSHOT_RESPONSE = ( + """{ "createStorageSnapshotResponse" : { + "StorageSnapshot" : { + "id": "21d7a92a-f15e-3f5b-b981-cb30697b8028", + "name": "DS1Snap1", + "usn": "21d7a92af15e3f5bb981cb30697b8028", + "lunusn": "12371e7c392b34b9ac43073b3c85f1d1", + "lunid": "12371e7c-392b-34b9-ac43-073b3c85f1d1", + "scsiEnabled": false + }} + }""") + +# A fake list volume iscsi service response of cloudbyte's elasticenter +FAKE_LIST_VOLUME_ISCSI_SERVICE_RESPONSE = ( + """{ "listVolumeiSCSIServiceResponse" : { + "count":1 , + "iSCSIService" : [{ + "id": "67ddcbf4-6887-3ced-8695-7b9cdffce885", + "volume_id": "c93df32e-3a99-3491-8e10-cf318a7f9b7f", + "ag_id": "4459248d-e9f1-3d2a-b7e8-b5d9ce587fc1", + "ig_id": "203e0235-1d5a-3130-9204-98e3f642a564", + "iqnname": "iqn.2014-06.acc1.openstacktsm:acc1DS1", + "authmethod": "None", + "status": true, + "usn": "92cfd601bc1f3fa78322c492099f3326", + "initialdigest": "Auto", + "queuedepth": "32", + "inqproduct": 0, + "inqrevision": 0, + "blocklength": "512B" + }] + }}""") + +# A fake clone dataset snapshot response of cloudbyte's elasticenter +FAKE_CLONE_DATASET_SNAPSHOT_RESPONSE = """{ "cloneDatasetSnapshot" : { + "filesystem" : { + "id": "dcd46a57-e3f4-3fc1-8dd8-2e658d9ebb11", + "name": "DS1Snap1clone1", + "simpleid": 21, + "type": "volume", + "revisionnumber": 1, + "path": "iqn.2014-06.acc1.openstacktsm:acc1DS1Snap1clone1", + "clusterid": "0ff44329-9a69-4611-bac2-6eaf1b08bb18", + "clusterstatus": "Online", + "Tsmid": "8146146e-f67b-3942-8074-3074599207a4", + "tsmType": "1", + "accountid": "86c5251a-9044-4690-b924-0d97627aeb8c", + "poolid": "73b567c0-e57d-37b5-b765-9d70725f59af", + "controllerid": "f1603e87-d1e6-3dcb-a549-7a6e77f82d86", + "groupid": "d73662ac-6db8-3b2c-981a-012af4e2f7bd", + "parentid": "81ebdcbb-f73b-3337-8f32-222820e6acb9", + "compression": "off", + "sync": "always", + "noofcopies": 1, + "recordsize": "4k", + "deduplication": "off", + "quota": "10G", + "unicode": "off", + "casesensitivity": "sensitive", + "readonly": false, + "nfsenabled": false, + "cifsenabled": false, + "iscsienabled": true, + "fcenabled": false, + "currentUsedSpace": 0, + "currentAvailableSpace": 10240, + "currentTotalSpace": 10240, + "currentThroughput": 0, + "currentIOPS": 0, + "currentLatency": 0, + "currentThrottle": 0, + "numericquota": 10240.0, + "status": "Online", + "managedstate": "Available", + "configurationstate": "sync", + "tsmName": "openstacktsm", + "ipaddress": "20.10.22.56", + "sitename": "site1", + "clustername": "HA1", + "controllerName": "node1", + "hapoolname": "devpool1", + "hapoolgrace": true, + "tsmgrace": true, + "tsmcontrolgrace": "false", + "accountname": "acc1", + "groupname": "QoS_DS1acc1openstacktsm", + "iops": "100", + "blocksize": "4k", + "throughput": "400", + "latency": "15", + "graceallowed": false, + "offlinenodes": "", + "tpcontrol": "true", + "iopscontrol": "true", + "tsmAvailIops": "700", + "tsmAvailTput": "2800", + "iqnname": "iqn.2014-06.acc1.openstacktsm:acc1DS1Snap1clone1", + "mountpoint": "acc1DS1Snap1clone1", + "blocklength": "512B", + "volumeaccessible": "true", + "localschedulecount": 0 + } + }}""" + +# A fake update filesystem response of cloudbyte's elasticenter +FAKE_UPDATE_FILE_SYSTEM_RESPONSE = """{ "updatefilesystemresponse" : { + "count":1 , + "filesystem" : [{ + "id": "92cfd601-bc1f-3fa7-8322-c492099f3326", + "name": "DS1", + "simpleid": 20, + "type": "volume", + "revisionnumber": 1, + "path": "iqn.2014-06.acc1.openstacktsm:acc1DS1", + "clusterid": "0ff44329-9a69-4611-bac2-6eaf1b08bb18", + "clusterstatus": "Online", + "Tsmid": "8146146e-f67b-3942-8074-3074599207a4", + "tsmType": "1", + "accountid": "86c5251a-9044-4690-b924-0d97627aeb8c", + "poolid": "73b567c0-e57d-37b5-b765-9d70725f59af", + "controllerid": "f1603e87-d1e6-3dcb-a549-7a6e77f82d86", + "groupid": "d73662ac-6db8-3b2c-981a-012af4e2f7bd", + "parentid": "81ebdcbb-f73b-3337-8f32-222820e6acb9", + "compression": "off", + "sync": "always", + "noofcopies": 1, + "recordsize": "4k", + "deduplication": "off", + "quota": "12G", + "unicode": "off", + "casesensitivity": "sensitive", + "readonly": false, + "nfsenabled": false, + "cifsenabled": false, + "iscsienabled": true, + "fcenabled": false, + "currentUsedSpace": 0, + "currentAvailableSpace": 10240, + "currentTotalSpace": 10240, + "currentThroughput": 0, + "currentIOPS": 0, + "currentLatency": 0, + "currentThrottle": 0, + "numericquota": 12288.0, + "status": "Online", + "managedstate": "Available", + "configurationstate": "sync", + "tsmName": "openstacktsm", + "ipaddress": "20.10.22.56", + "sitename": "site1", + "clustername": "HA1", + "controllerName": "node1", + "hapoolname": "devpool1", + "hapoolgrace": true, + "tsmgrace": true, + "tsmcontrolgrace": "false", + "accountname": "acc1", + "groupname": "QoS_DS1acc1openstacktsm", + "iops": "100", + "blocksize": "4k", + "throughput": "400", + "latency": "15", + "graceallowed": false, + "offlinenodes": "", + "tpcontrol": "true", + "iopscontrol": "true", + "tsmAvailIops": "700", + "tsmAvailTput": "2800", + "iqnname": "iqn.2014-06.acc1.openstacktsm:acc1DS1", + "mountpoint": "acc1DS1", + "blocklength": "512B", + "volumeaccessible": "true", + "localschedulecount": 0 + }] + }}""" + +# A fake update QOS group response of cloudbyte's elasticenter +FAKE_UPDATE_QOS_GROUP_RESPONSE = """{ "updateqosresponse" : { + "count":1 , + "qosgroup" : [{ + "id": "d73662ac-6db8-3b2c-981a-012af4e2f7bd", + "name": "QoS_DS1acc1openstacktsm", + "tsmid": "8146146e-f67b-3942-8074-3074599207a4", + "controllerid": "f1603e87-d1e6-3dcb-a549-7a6e77f82d86", + "poolid": "73b567c0-e57d-37b5-b765-9d70725f59af", + "parentid": "81ebdcbb-f73b-3337-8f32-222820e6acb9", + "tsmName": "openstacktsm", + "offlinenodes": "", + "sitename": "site1", + "clustername": "HA1", + "controllerName": "node1", + "clusterstatus": "Online", + "currentThroughput": 0, + "currentIOPS": 0, + "currentLatency": 0, + "currentThrottle": 0, + "iopsvalue": "(0/101)", + "throughputvalue": "(0/404)", + "iops": "101", + "iopscontrol": "true", + "throughput": "404", + "tpcontrol": "true", + "blocksize": "4k", + "latency": "15", + "graceallowed": true, + "type": "1", + "revisionnumber": 2, + "managedstate": "Available", + "configurationstate": "sync", + "status": "Online", + "standardproviops": 0, + "operatingblocksize": 0, + "operatingcachehit": 0, + "operatingiops": 0, + "standardoperatingiops": 0 + }] + }}""" + + +# This dict maps the http commands of elasticenter +# with its respective fake responses +MAP_COMMAND_TO_FAKE_RESPONSE = {} + +MAP_COMMAND_TO_FAKE_RESPONSE['deleteFileSystem'] = ( + json.loads(FAKE_DELETE_FILE_SYSTEM_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["listFileSystem"] = ( + json.loads(FAKE_LIST_FILE_SYSTEM_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["deleteSnapshot"] = ( + json.loads(FAKE_DELETE_STORAGE_SNAPSHOT_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["listStorageSnapshots"] = ( + json.loads(FAKE_LIST_STORAGE_SNAPSHOTS_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["updateVolumeiSCSIService"] = ( + json.loads(FAKE_UPDATE_VOLUME_ISCSI_SERVICE_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["createStorageSnapshot"] = ( + json.loads(FAKE_CREATE_STORAGE_SNAPSHOT_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["listAccount"] = ( + json.loads(FAKE_LIST_ACCOUNT_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["listTsm"] = ( + json.loads(FAKE_LIST_TSM_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["addQosGroup"] = ( + json.loads(FAKE_ADD_QOS_GROUP_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["queryAsyncJobResult"] = ( + json.loads(FAKE_QUERY_ASYNC_JOB_RESULT_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["createVolume"] = ( + json.loads(FAKE_CREATE_VOLUME_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["listVolumeiSCSIService"] = ( + json.loads(FAKE_LIST_VOLUME_ISCSI_SERVICE_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE["listiSCSIInitiator"] = ( + json.loads(FAKE_LIST_ISCSI_INITIATOR_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE['cloneDatasetSnapshot'] = ( + json.loads(FAKE_CLONE_DATASET_SNAPSHOT_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE['updateFileSystem'] = ( + json.loads(FAKE_UPDATE_FILE_SYSTEM_RESPONSE)) +MAP_COMMAND_TO_FAKE_RESPONSE['updateQosGroup'] = ( + json.loads(FAKE_UPDATE_QOS_GROUP_RESPONSE)) + +# This dict maps the http commands of elasticenter +# with its respective fake json responses +MAP_COMMAND_TO_FAKE_JSON_RESPONSE = {} + +MAP_COMMAND_TO_FAKE_JSON_RESPONSE["listTsm"] = FAKE_LIST_TSM_RESPONSE + + +class CloudByteISCSIDriverTestCase(testtools.TestCase): + + def setUp(self): + super(CloudByteISCSIDriverTestCase, self).setUp() + self._configure_driver() + + def _configure_driver(self): + + configuration = conf.Configuration(None, None) + + # initialize the elasticenter iscsi driver + self.driver = CloudByteISCSIDriver(configuration=configuration) + + # override some parts of driver configuration + self.driver.configuration.tsm_name = 'openstack' + self.driver.configuration.cb_account_name = 'CustomerA' + + def _side_effect_api_req(self, cmd, params, version='1.0'): + """This is a side effect function. + + The return value is determined based on cmd argument. + The signature matches exactly with the method it tries + to mock. + """ + return MAP_COMMAND_TO_FAKE_RESPONSE[cmd] + + def _side_effect_api_req_to_create_vol(self, cmd, params, version='1.0'): + """This is a side effect function.""" + if cmd == 'createVolume': + return {} + + return MAP_COMMAND_TO_FAKE_RESPONSE[cmd] + + def _side_effect_api_req_to_query_asyncjob( + self, cmd, params, version='1.0'): + """This is a side effect function.""" + + if cmd == 'queryAsyncJobResult': + return {'queryasyncjobresultresponse': {'jobstatus': 0}} + + return MAP_COMMAND_TO_FAKE_RESPONSE[cmd] + + def _side_effect_api_req_to_list_tsm(self, cmd, params, version='1.0'): + """This is a side effect function.""" + if cmd == 'listTsm': + return {} + + return MAP_COMMAND_TO_FAKE_RESPONSE[cmd] + + def _side_effect_api_req_to_list_filesystem( + self, cmd, params, version='1.0'): + """This is a side effect function.""" + if cmd == 'listFileSystem': + return {} + + return MAP_COMMAND_TO_FAKE_RESPONSE[cmd] + + def _side_effect_api_req_to_list_vol_iscsi_service( + self, cmd, params, version='1.0'): + """This is a side effect function.""" + if cmd == 'listVolumeiSCSIService': + return {} + + return MAP_COMMAND_TO_FAKE_RESPONSE[cmd] + + def _side_effect_api_req_to_list_iscsi_initiator( + self, cmd, params, version='1.0'): + """This is a side effect function.""" + if cmd == 'listiSCSIInitiator': + return {} + + return MAP_COMMAND_TO_FAKE_RESPONSE[cmd] + + def _side_effect_create_vol_from_snap(self, cloned_volume, snapshot): + """This is a side effect function.""" + return {} + + def _side_effect_create_snapshot(self, snapshot): + """This is a side effect function.""" + model_update = {} + model_update['provider_id'] = "devpool1/acc1openstacktsm/DS1@DS1Snap1" + return model_update + + def _side_effect_get_connection(self, host, url): + """This is a side effect function.""" + + return_obj = {} + + return_obj['http_status'] = 200 + + # mock the response data + return_obj['data'] = MAP_COMMAND_TO_FAKE_RESPONSE['listTsm'] + return_obj['error'] = None + + return return_obj + + def _side_effect_get_err_connection(self, host, url): + """This is a side effect function.""" + + return_obj = {} + + return_obj['http_status'] = 500 + + # mock the response data + return_obj['data'] = None + return_obj['error'] = "Http status: 500, Error: Elasticenter " + "is not available." + + return return_obj + + def _side_effect_get_err_connection2(self, host, url): + """This is a side effect function.""" + + msg = ("Error executing CloudByte API %(cmd)s , Error: %(err)s" % + {'cmd': 'MockTest', 'err': 'Error'}) + raise exception.VolumeBackendAPIException(msg) + + def _get_fake_volume_id(self): + + # Get the filesystems + fs_list = MAP_COMMAND_TO_FAKE_RESPONSE['listFileSystem'] + filesystems = fs_list['listFilesystemResponse']['filesystem'] + + # Get the volume id from the first filesystem + volume_id = filesystems[0]['id'] + + return volume_id + + @mock.patch.object(CloudByteISCSIDriver, + '_execute_and_get_response_details') + def test_api_request_for_cloudbyte(self, mock_conn): + + # Test - I + + # configure the mocks with respective side-effects + mock_conn.side_effect = self._side_effect_get_connection + + # run the test + data = self.driver._api_request_for_cloudbyte('listTsm', {}) + + # assert the data attributes + self.assertEqual(1, data['listTsmResponse']['count']) + + # Test - II + + # configure the mocks with side-effects + mock_conn.reset_mock() + mock_conn.side_effect = self._side_effect_get_err_connection + + # run the test + with ExpectedException( + exception.VolumeBackendAPIException, + 'Bad or unexpected response from the storage volume ' + 'backend API: Failed to execute CloudByte API'): + self.driver._api_request_for_cloudbyte('listTsm', {}) + + # Test - III + + # configure the mocks with side-effects + mock_conn.reset_mock() + mock_conn.side_effect = self._side_effect_get_err_connection2 + + # run the test + with ExpectedException( + exception.VolumeBackendAPIException, + 'Error executing CloudByte API'): + self.driver._api_request_for_cloudbyte('listTsm', {}) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_delete_volume(self, mock_api_req): + + # prepare the dependencies + fake_volume_id = self._get_fake_volume_id() + volume = {'id': fake_volume_id, 'provider_id': fake_volume_id} + + # Test-I + + mock_api_req.side_effect = self._side_effect_api_req + + # run the test + self.driver.delete_volume(volume) + + # assert that 2 api calls were invoked + self.assertEqual(2, mock_api_req.call_count) + + # Test-II + + # reset & re-configure mock + volume['provider_id'] = None + mock_api_req.reset_mock() + mock_api_req.side_effect = self._side_effect_api_req + + # run the test + self.driver.delete_volume(volume) + + # assert that no api calls were invoked + self.assertEqual(0, mock_api_req.call_count) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_delete_snapshot(self, mock_api_req): + + snapshot = { + 'id': 'SomeID', + 'provider_id': 'devpool1/acc1openstacktsm/DS1@DS1Snap1', + 'display_name': 'DS1Snap1', + 'volume_id': 'SomeVol', + 'volume': { + 'display_name': 'DS1' + } + + } + + # Test - I + + # now run the test + self.driver.delete_snapshot(snapshot) + + # assert that 1 api call was invoked + self.assertEqual(1, mock_api_req.call_count) + + # Test - II + + # reconfigure the dependencies + snapshot['provider_id'] = None + + # reset & reconfigure the mock + mock_api_req.reset_mock() + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test + self.driver.delete_snapshot(snapshot) + + # assert that no api calls were invoked + self.assertEqual(0, mock_api_req.call_count) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_create_snapshot(self, mock_api_req): + + # prepare the dependencies + fake_volume_id = self._get_fake_volume_id() + + snapshot = { + 'id': 'SomeID', + 'display_name': 'DS1Snap1', + 'volume_id': 'SomeVol', + 'volume': { + 'display_name': 'DS1', + 'provider_id': fake_volume_id + + } + } + + # Test - I + + # configure the mocks with respective side-effects + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test + self.driver.create_snapshot(snapshot) + + # assert that 2 api calls were invoked + self.assertEqual(2, mock_api_req.call_count) + + # Test - II + + # reconfigure the dependencies + snapshot['volume']['provider_id'] = None + + # reset & reconfigure the mock + mock_api_req.reset_mock() + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test & assert the exception + with ExpectedException( + exception.VolumeBackendAPIException, + 'Bad or unexpected response from the storage volume ' + 'backend API: Failed to create snapshot'): + self.driver.create_snapshot(snapshot) + + # assert that no api calls were invoked + self.assertEqual(0, mock_api_req.call_count) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_create_volume(self, mock_api_req): + + # prepare the dependencies + fake_volume_id = self._get_fake_volume_id() + + volume = { + 'id': fake_volume_id, + 'size': 22 + } + + # Test - I + + # configure the mocks with respective side-effects + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test + provider_details = self.driver.create_volume(volume) + + # assert equality checks for certain configuration attributes + self.assertEqual( + 'openstack', self.driver.configuration.tsm_name) + self.assertEqual( + 'CustomerA', self.driver.configuration.cb_account_name) + self.assertThat( + provider_details['provider_location'], + Contains('172.16.50.35:3260')) + + # assert that 9 api calls were invoked + self.assertEqual(9, mock_api_req.call_count) + + # Test - II + + # reconfigure the dependencies + volume['id'] = 'NotExists' + del volume['size'] + + # reset & reconfigure the mock + mock_api_req.reset_mock() + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test & assert the exception + with ExpectedException( + exception.VolumeBackendAPIException, + "Bad or unexpected response from the storage volume " + "backend API: Volume \[NotExists\] not found in " + "CloudByte storage."): + self.driver.create_volume(volume) + + # Test - III + + # reconfigure the dependencies + volume['id'] = 'abc' + + # reset the mocks + mock_api_req.reset_mock() + + # configure or re-configure the mocks + mock_api_req.side_effect = self._side_effect_api_req_to_create_vol + + # now run the test & assert the exception + with ExpectedException( + exception.VolumeBackendAPIException, + 'Bad or unexpected response from the storage volume ' + 'backend API: Null response received while ' + 'creating volume'): + self.driver.create_volume(volume) + + # Test - IV + + # reconfigure the dependencies + # reset the mocks + mock_api_req.reset_mock() + + # configure or re-configure the mocks + mock_api_req.side_effect = self._side_effect_api_req_to_list_filesystem + + # now run the test + with ExpectedException( + exception.VolumeBackendAPIException, + "Bad or unexpected response from the storage volume " + "backend API: Null response received from CloudByte's " + "list filesystem."): + self.driver.create_volume(volume) + + # Test - VI + + volume['id'] = fake_volume_id + # reconfigure the dependencies + # reset the mocks + mock_api_req.reset_mock() + + # configure or re-configure the mocks + mock_api_req.side_effect = ( + self._side_effect_api_req_to_list_vol_iscsi_service) + + # now run the test + with ExpectedException( + exception.VolumeBackendAPIException, + "Bad or unexpected response from the storage volume " + "backend API: Null response received from CloudByte's " + "list volume iscsi service."): + self.driver.create_volume(volume) + + # Test - VII + + # reconfigure the dependencies + # reset the mocks + mock_api_req.reset_mock() + + # configure or re-configure the mocks + mock_api_req.side_effect = ( + self._side_effect_api_req_to_list_iscsi_initiator) + + # now run the test + with ExpectedException( + exception.VolumeBackendAPIException, + "Bad or unexpected response from the storage volume " + "backend API: Null response received from CloudByte's " + "list iscsi initiators."): + self.driver.create_volume(volume) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + @mock.patch.object(CloudByteISCSIDriver, + 'create_volume_from_snapshot') + @mock.patch.object(CloudByteISCSIDriver, + 'create_snapshot') + def test_create_cloned_volume(self, mock_create_snapshot, + mock_create_vol_from_snap, mock_api_req): + + # prepare the input test data + fake_volume_id = self._get_fake_volume_id() + + src_volume = {'display_name': 'DS1Snap1'} + + cloned_volume = { + 'source_volid': fake_volume_id, + 'id': 'SomeNewID', + 'display_name': 'CloneOfDS1Snap1' + } + + # Test - I + + # configure the mocks with respective sideeffects + mock_api_req.side_effect = self._side_effect_api_req + mock_create_vol_from_snap.side_effect = ( + self._side_effect_create_vol_from_snap) + mock_create_snapshot.side_effect = ( + self._side_effect_create_snapshot) + + # now run the test + self.driver.create_cloned_volume(cloned_volume, src_volume) + + # assert that n api calls were invoked + self.assertEqual(0, mock_api_req.call_count) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_create_volume_from_snapshot(self, mock_api_req): + + # prepare the input test data + fake_volume_id = self._get_fake_volume_id() + + snapshot = { + 'volume_id': fake_volume_id, + 'provider_id': 'devpool1/acc1openstacktsm/DS1@DS1Snap1', + 'id': 'SomeSnapID', + 'volume': { + 'provider_id': fake_volume_id + } + } + + cloned_volume = { + 'display_name': 'CloneOfDS1Snap1', + 'id': 'ClonedVolID' + } + + # Test - I + + # configure the mocks with respective side-effects + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test + self.driver.create_volume_from_snapshot(cloned_volume, snapshot) + + # assert n api calls were invoked + self.assertEqual(1, mock_api_req.call_count) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_extend_volume(self, mock_api_req): + + # prepare the input test data + fake_volume_id = self._get_fake_volume_id() + + volume = { + 'id': 'SomeID', + 'provider_id': fake_volume_id + } + + new_size = '2' + + # Test - I + + # configure the mock with respective side-effects + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test + self.driver.extend_volume(volume, new_size) + + # assert n api calls were invoked + self.assertEqual(1, mock_api_req.call_count) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_create_export(self, mock_api_req): + + # prepare the input test data + + # configure the mocks with respective side-effects + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test + model_update = self.driver.create_export({}, {}) + + # assert the result + self.assertEqual(None, model_update['provider_auth']) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_ensure_export(self, mock_api_req): + + # prepare the input test data + + # configure the mock with respective side-effects + mock_api_req.side_effect = self._side_effect_api_req + + # now run the test + model_update = self.driver.ensure_export({}, {}) + + # assert the result to have a provider_auth attribute + self.assertEqual(None, model_update['provider_auth']) + + @mock.patch.object(CloudByteISCSIDriver, + '_api_request_for_cloudbyte') + def test_get_volume_stats(self, mock_api_req): + + # prepare the input test data + + # configure the mock with a side-effect + mock_api_req.side_effect = self._side_effect_api_req + + # Test - I + + # run the test + vol_stats = self.driver.get_volume_stats() + + # assert 0 api calls were invoked + self.assertEqual(0, mock_api_req.call_count) + + # Test - II + + # run the test with refresh as True + vol_stats = self.driver.get_volume_stats(refresh=True) + + # assert n api calls were invoked + self.assertEqual(1, mock_api_req.call_count) + + # assert the result attributes with respective values + self.assertEqual(0, vol_stats['reserved_percentage']) + self.assertEqual('CloudByte', vol_stats['vendor_name']) + self.assertEqual('iSCSI', vol_stats['storage_protocol']) + + # Test - III + + # configure the mocks with side-effect + mock_api_req.reset_mock() + mock_api_req.side_effect = self._side_effect_api_req_to_list_tsm + + # run the test with refresh as True + with ExpectedException( + exception.VolumeBackendAPIException, + "Bad or unexpected response from the storage volume " + "backend API: No response was received from CloudByte " + "storage list tsm API call."): + self.driver.get_volume_stats(refresh=True) diff --git a/cinder/volume/drivers/cloudbyte/__init__.py b/cinder/volume/drivers/cloudbyte/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cinder/volume/drivers/cloudbyte/cloudbyte.py b/cinder/volume/drivers/cloudbyte/cloudbyte.py new file mode 100644 index 000000000..c33b9df0a --- /dev/null +++ b/cinder/volume/drivers/cloudbyte/cloudbyte.py @@ -0,0 +1,951 @@ +# Copyright 2015 CloudByte Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import httplib +import json +import time +import urllib + +import six + +from cinder import exception +from cinder.i18n import _, _LE, _LI +from cinder.openstack.common import log as logging +from cinder.openstack.common import loopingcall +from cinder.volume.drivers.cloudbyte.options import ( + cloudbyte_create_volume_opts +) +from cinder.volume.drivers.cloudbyte.options import cloudbyte_add_qosgroup_opts +from cinder.volume.drivers.cloudbyte.options import cloudbyte_connection_opts +from cinder.volume.drivers.san import san + +LOG = logging.getLogger(__name__) + + +class CloudByteISCSIDriver(san.SanISCSIDriver): + """CloudByte ISCSI Driver. + + Version history: + 1.0.0 - Initial driver + """ + + VERSION = '1.0.0' + volume_stats = {} + + def __init__(self, *args, **kwargs): + super(CloudByteISCSIDriver, self).__init__(*args, **kwargs) + self.configuration.append_config_values(cloudbyte_add_qosgroup_opts) + self.configuration.append_config_values(cloudbyte_create_volume_opts) + self.configuration.append_config_values(cloudbyte_connection_opts) + self.get_volume_stats() + + def _get_url(self, cmd, params, apikey): + """Will prepare URL that connects to CloudByte.""" + + if params is None: + params = {} + + params['command'] = cmd + params['response'] = 'json' + + sanitized_params = {} + + for key in params: + value = params[key] + if value is not None: + sanitized_params[key] = six.text_type(value) + + sanitized_params = urllib.urlencode(sanitized_params) + url = ('/client/api?%s' % sanitized_params) + + LOG.debug("CloudByte URL to be executed: [%s].", url) + + # Add the apikey + api = {} + api['apiKey'] = apikey + url = url + '&' + urllib.urlencode(api) + + return url + + def _extract_http_error(self, error_data): + # Extract the error message from error_data + error_msg = "" + + # error_data is a single key value dict + for key, value in error_data.iteritems(): + error_msg = value.get('errortext') + + return error_msg + + def _execute_and_get_response_details(self, host, url): + """Will prepare response after executing an http request.""" + + res_details = {} + try: + # Prepare the connection + connection = httplib.HTTPSConnection(host) + # Make the connection + connection.request('GET', url) + # Extract the response as the connection was successful + response = connection.getresponse() + # Read the response + data = response.read() + # Transform the json string into a py object + data = json.loads(data) + # Extract http error msg if any + error_details = None + if response.status != 200: + error_details = self._extract_http_error(data) + + # Prepare the return object + res_details['data'] = data + res_details['error'] = error_details + res_details['http_status'] = response.status + + finally: + connection.close() + LOG.debug("CloudByte connection was closed successfully.") + + return res_details + + def _api_request_for_cloudbyte(self, cmd, params, version=None): + """Make http calls to CloudByte.""" + LOG.debug("Executing CloudByte API for command [%s].", cmd) + + if version is None: + version = CloudByteISCSIDriver.VERSION + + # Below is retrieved from /etc/cinder/cinder.conf + apikey = self.configuration.cb_apikey + + if apikey is None: + msg = (_("API key is missing for CloudByte driver.")) + raise exception.VolumeBackendAPIException(data=msg) + + host = self.configuration.san_ip + + # Construct the CloudByte URL with query params + url = self._get_url(cmd, params, apikey) + + data = {} + error_details = None + http_status = None + + try: + # Execute CloudByte API & frame the response + res_obj = self._execute_and_get_response_details(host, url) + + data = res_obj['data'] + error_details = res_obj['error'] + http_status = res_obj['http_status'] + + except httplib.HTTPException as ex: + msg = (_("Error executing CloudByte API [%(cmd)s], " + "Error: %(err)s.") % + {'cmd': cmd, 'err': ex}) + raise exception.VolumeBackendAPIException(data=msg) + + # Check if it was an error response from CloudByte + if http_status != 200: + msg = (_("Failed to execute CloudByte API [%(cmd)s]." + " Http status: %(status)s," + " Error: %(error)s.") % + {'cmd': cmd, 'status': http_status, + 'error': error_details}) + raise exception.VolumeBackendAPIException(data=msg) + + LOG.info(_LI("CloudByte API executed successfully for command [%s]."), + cmd) + + return data + + def _request_tsm_details(self, account_id): + params = {"accountid": account_id} + + # List all CloudByte tsm + data = self._api_request_for_cloudbyte("listTsm", params) + return data + + def _override_params(self, default_dict, filtered_user_dict): + """Override the default config values with user provided values. + """ + + if filtered_user_dict is None: + # Nothing to override + return default_dict + + for key, value in default_dict.iteritems(): + # Fill the user dict with default options based on condition + if filtered_user_dict.get(key) is None and value is not None: + filtered_user_dict[key] = value + + return filtered_user_dict + + def _add_qos_group_request(self, volume, tsmid, volume_name): + # Get qos related params from configuration + params = self.configuration.cb_add_qosgroup + + if params is None: + params = {} + + params['name'] = "QoS_" + volume_name + params['tsmid'] = tsmid + + data = self._api_request_for_cloudbyte("addQosGroup", params) + return data + + def _create_volume_request(self, volume, datasetid, qosgroupid, + tsmid, volume_name): + + size = volume.get('size') + quotasize = six.text_type(size) + "G" + + # Prepare the user input params + params = { + "datasetid": datasetid, + "name": volume_name, + "qosgroupid": qosgroupid, + "tsmid": tsmid, + "quotasize": quotasize + } + + # Get the additional params from configuration + params = self._override_params(self.configuration.cb_create_volume, + params) + + data = self._api_request_for_cloudbyte("createVolume", params) + return data + + def _queryAsyncJobResult_request(self, jobid): + async_cmd = "queryAsyncJobResult" + params = { + "jobId": jobid, + } + data = self._api_request_for_cloudbyte(async_cmd, params) + return data + + def _get_tsm_details(self, data, tsm_name): + # Filter required tsm's details + tsms = data['listTsmResponse']['listTsm'] + tsmdetails = {} + for tsm in tsms: + if tsm['name'] == tsm_name: + tsmdetails['datasetid'] = tsm['datasetid'] + tsmdetails['tsmid'] = tsm['id'] + break + + return tsmdetails + + def _wait_for_volume_creation(self, volume_response, cb_volume_name): + """Given the job wait for it to complete.""" + + vol_res = volume_response.get('createvolumeresponse') + + if vol_res is None: + msg = _("Null response received while creating volume [%s] " + "at CloudByte storage.") % cb_volume_name + raise exception.VolumeBackendAPIException(data=msg) + + jobid = vol_res.get('jobid') + + if jobid is None: + msg = _("Jobid not found in CloudByte's " + "create volume [%s] response.") % cb_volume_name + raise exception.VolumeBackendAPIException(data=msg) + + def _retry_check_for_volume_creation(): + """Called at an interval till the volume is created.""" + + retries = kwargs['retries'] + max_retries = kwargs['max_retries'] + jobid = kwargs['jobid'] + cb_vol = kwargs['cb_vol'] + + # Query the CloudByte storage with this jobid + volume_response = self._queryAsyncJobResult_request(jobid) + + result_res = None + if volume_response is not None: + result_res = volume_response.get('queryasyncjobresultresponse') + + if volume_response is None or result_res is None: + msg = _( + "Null response received while querying " + "for create volume job [%s] " + "at CloudByte storage.") % jobid + raise exception.VolumeBackendAPIException(data=msg) + + status = result_res.get('jobstatus') + + if status == 1: + LOG.info(_LI("Volume [%s] created successfully in " + "CloudByte storage."), cb_vol) + raise loopingcall.LoopingCallDone() + + elif retries == max_retries: + # All attempts exhausted + LOG.error(_LE("Error in creating volume [%(vol)s] in " + "CloudByte storage. " + "Exhausted all [%(max)s] attempts."), + {'vol': cb_vol, 'max': retries}) + raise loopingcall.LoopingCallDone(retvalue=False) + + else: + retries += 1 + kwargs['retries'] = retries + LOG.debug("Wait for volume [%(vol)s] creation, " + "retry [%(retry)s] of [%(max)s].", + {'vol': cb_vol, + 'retry': retries, + 'max': max_retries}) + + retry_interval = ( + self.configuration.cb_confirm_volume_create_retry_interval) + + max_retries = ( + self.configuration.cb_confirm_volume_create_retries) + + kwargs = {'retries': 0, + 'max_retries': max_retries, + 'jobid': jobid, + 'cb_vol': cb_volume_name} + + timer = loopingcall.FixedIntervalLoopingCall( + _retry_check_for_volume_creation) + timer.start(interval=retry_interval).wait() + + def _get_volume_id_from_response(self, cb_volumes, volume_name): + """Search the volume in CloudByte storage.""" + + vol_res = cb_volumes.get('listFilesystemResponse') + + if vol_res is None: + msg = _("Null response received from CloudByte's " + "list filesystem.") + raise exception.VolumeBackendAPIException(data=msg) + + volumes = vol_res.get('filesystem') + + if volumes is None: + msg = _('No volumes found in CloudByte storage.') + raise exception.VolumeBackendAPIException(data=msg) + + volume_id = None + + for vol in volumes: + if vol['name'] == volume_name: + volume_id = vol['id'] + break + + if volume_id is None: + msg = _("Volume [%s] not found in CloudByte " + "storage.") % volume_name + raise exception.VolumeBackendAPIException(data=msg) + + return volume_id + + def _get_qosgroupid_id_from_response(self, cb_volumes, volume_id): + volumes = cb_volumes['listFilesystemResponse']['filesystem'] + qosgroup_id = None + + for vol in volumes: + if vol['id'] == volume_id: + qosgroup_id = vol['groupid'] + break + + return qosgroup_id + + def _build_provider_details_from_volume(self, volume): + model_update = {} + + model_update['provider_location'] = ( + '%s %s %s' % (volume['ipaddress'] + ':3260', volume['iqnname'], 0) + ) + + # Will provide CHAP Authentication on forthcoming patches/release + model_update['provider_auth'] = None + + model_update['provider_id'] = volume['id'] + + LOG.debug("CloudByte volume [%(vol)s] properties: [%(props)s].", + {'vol': volume['iqnname'], 'props': model_update}) + + return model_update + + def _build_provider_details_from_response(self, cb_volumes, volume_name): + """Get provider information.""" + + model_update = {} + volumes = cb_volumes['listFilesystemResponse']['filesystem'] + + for vol in volumes: + if vol['name'] == volume_name: + model_update = self._build_provider_details_from_volume(vol) + break + + return model_update + + def _get_initiator_group_id_from_response(self, data): + """Find iSCSI initiator group id.""" + + ig_list_res = data.get('listInitiatorsResponse') + + if ig_list_res is None: + msg = _("Null response received from CloudByte's " + "list iscsi initiators.") + raise exception.VolumeBackendAPIException(data=msg) + + ig_list = ig_list_res.get('initiator') + + if ig_list is None: + msg = _('No iscsi initiators were found in CloudByte.') + raise exception.VolumeBackendAPIException(data=msg) + + ig_id = None + + for ig in ig_list: + if ig.get('initiatorgroup') == 'ALL': + ig_id = ig['id'] + break + + return ig_id + + def _get_iscsi_service_id_from_response(self, volume_id, data): + iscsi_service_res = data.get('listVolumeiSCSIServiceResponse') + + if iscsi_service_res is None: + msg = _("Null response received from CloudByte's " + "list volume iscsi service.") + raise exception.VolumeBackendAPIException(data=msg) + + iscsi_service_list = iscsi_service_res.get('iSCSIService') + + if iscsi_service_list is None: + msg = _('No iscsi services found in CloudByte storage.') + raise exception.VolumeBackendAPIException(data=msg) + + iscsi_id = None + + for iscsi_service in iscsi_service_list: + if iscsi_service['volume_id'] == volume_id: + iscsi_id = iscsi_service['id'] + break + + if iscsi_id is None: + msg = _("No iscsi service found for CloudByte " + "volume [%s].") % volume_id + raise exception.VolumeBackendAPIException(data=msg) + else: + return iscsi_id + + def _request_update_iscsi_service(self, iscsi_id, ig_id): + params = { + "id": iscsi_id, + "igid": ig_id + } + + self._api_request_for_cloudbyte( + 'updateVolumeiSCSIService', params) + + def _get_cb_snapshot_path(self, snapshot, volume_id): + """Find CloudByte snapshot path.""" + + params = {"id": volume_id} + + # List all snapshot from CloudByte + cb_snapshots_list = self._api_request_for_cloudbyte( + 'listStorageSnapshots', params) + + # Filter required snapshot from list + cb_snap_res = cb_snapshots_list.get('listDatasetSnapshotsResponse') + + cb_snapshot = {} + if cb_snap_res is not None: + cb_snapshot = cb_snap_res.get('snapshot') + + path = None + + # Filter snapshot path + for snap in cb_snapshot: + if snap['name'] == snapshot['display_name']: + path = snap['path'] + break + + return path + + def _get_account_id_from_name(self, account_name): + params = {} + data = self._api_request_for_cloudbyte("listAccount", params) + accounts = data["listAccountResponse"]["account"] + + account_id = None + for account in accounts: + if account.get("name") == account_name: + account_id = account.get("id") + break + + if account_id is None: + msg = _("Failed to get CloudByte account details " + "for account [%s].") % account_name + raise exception.VolumeBackendAPIException(data=msg) + + return account_id + + def _search_volume_id(self, cb_volumes, cb_volume_id): + """Search the volume in CloudByte.""" + + volumes_res = cb_volumes.get('listFilesystemResponse') + + if volumes_res is None: + msg = _("No response was received from CloudByte's " + "list filesystem api call.") + raise exception.VolumeBackendAPIException(data=msg) + + volumes = volumes_res.get('filesystem') + + if volumes is None: + msg = _("No volume was found at CloudByte storage.") + raise exception.VolumeBackendAPIException(data=msg) + + volume_id = None + + for vol in volumes: + if vol['id'] == cb_volume_id: + volume_id = vol['id'] + break + + return volume_id + + def _generate_clone_name(self): + """Generates clone name when it is not provided.""" + + clone_name = ("clone_" + time.strftime("%d%m%Y") + + time.strftime("%H%M%S")) + return clone_name + + def _generate_snapshot_name(self): + """Generates snapshot_name when it is not provided.""" + + snapshot_name = ("snap_" + time.strftime("%d%m%Y") + + time.strftime("%H%M%S")) + return snapshot_name + + def _get_storage_info(self, tsmname): + """Get CloudByte TSM that is associated with OpenStack backend.""" + + # List all TSMs from CloudByte storage + tsm_list = self._api_request_for_cloudbyte('listTsm', params={}) + + tsm_details_res = tsm_list.get('listTsmResponse') + + if tsm_details_res is None: + msg = _("No response was received from CloudByte storage " + "list tsm API call.") + raise exception.VolumeBackendAPIException(data=msg) + + tsm_details = tsm_details_res.get('listTsm') + + data = {} + flag = 0 + # Filter required TSM and get storage info + for tsms in tsm_details: + if tsms['name'] == tsmname: + flag = 1 + storage_buckets = {} + storage_buckets = tsms['storageBuckets'] + quota = 0 + for bucket in storage_buckets: + quota = bucket['quota'] + break + + data['total_capacity_gb'] = quota + data['free_capacity_gb'] = ( + int(tsms['availablequota']) / 1000) + + # TSM not found in CloudByte storage + if flag == 0: + LOG.error(_LE("TSM [%s] not found in CloudByte storage."), tsmname) + data['total_capacity_gb'] = 0 + data['free_capacity_gb'] = 0 + + return data + + def create_volume(self, volume): + + tsm_name = self.configuration.cb_tsm_name + account_name = self.configuration.cb_account_name + + # Get account id of this account + account_id = self._get_account_id_from_name(account_name) + + # Set backend storage volume name using OpenStack volume id + cb_volume_name = volume['id'].replace("-", "") + + LOG.debug("Will create a volume [%(cb_vol)s] in TSM [%(tsm)s] " + "at CloudByte storage w.r.t " + "OpenStack volume [%(stack_vol)s].", + {'cb_vol': cb_volume_name, + 'stack_vol': volume.get('id'), + 'tsm': tsm_name}) + + tsm_data = self._request_tsm_details(account_id) + tsm_details = self._get_tsm_details(tsm_data, tsm_name) + + # Send request to create a qos group before creating a volume + LOG.debug("Creating qos group for CloudByte volume [%s].", + cb_volume_name) + qos_data = self._add_qos_group_request( + volume, tsm_details.get('tsmid'), cb_volume_name) + + # Extract the qos group id from response + qosgroupid = qos_data['addqosgroupresponse']['qosgroup']['id'] + + LOG.debug("Successfully created qos group for CloudByte volume [%s].", + cb_volume_name) + + # Send a create volume request to CloudByte API + vol_data = self._create_volume_request( + volume, tsm_details.get('datasetid'), qosgroupid, + tsm_details.get('tsmid'), cb_volume_name) + + # Since create volume is an async call; + # need to confirm the creation before proceeding further + self._wait_for_volume_creation(vol_data, cb_volume_name) + + # Fetch iscsi id + cb_volumes = self._api_request_for_cloudbyte( + 'listFileSystem', params={}) + volume_id = self._get_volume_id_from_response(cb_volumes, + cb_volume_name) + + params = {"storageid": volume_id} + + iscsi_service_data = self._api_request_for_cloudbyte( + 'listVolumeiSCSIService', params) + iscsi_id = self._get_iscsi_service_id_from_response( + volume_id, iscsi_service_data) + + # Fetch the initiator group ID + params = {"accountid": account_id} + + iscsi_initiator_data = self._api_request_for_cloudbyte( + 'listiSCSIInitiator', params) + ig_id = self._get_initiator_group_id_from_response( + iscsi_initiator_data) + + LOG.debug("Updating iscsi service for CloudByte volume [%s].", + cb_volume_name) + + # Update the iscsi service with above fetched iscsi_id & ig_id + self._request_update_iscsi_service(iscsi_id, ig_id) + + LOG.debug("CloudByte volume [%(vol)s] updated with " + "iscsi id [%(iscsi)s] and ig id [%(ig)s].", + {'vol': cb_volume_name, 'iscsi': iscsi_id, 'ig': ig_id}) + + # Provide the model after successful completion of above steps + provider = self._build_provider_details_from_response( + cb_volumes, cb_volume_name) + + LOG.info(_LI("Successfully created a CloudByte volume [%(cb_vol)s] " + "w.r.t OpenStack volume [%(stack_vol)s]."), + {'cb_vol': cb_volume_name, 'stack_vol': volume.get('id')}) + + return provider + + def delete_volume(self, volume): + + params = {} + + # OpenStack source volume id + source_volume_id = volume['id'] + + # CloudByte volume id equals OpenStack volume's provider_id + cb_volume_id = volume.get('provider_id') + + LOG.debug("Will delete CloudByte volume [%(cb_vol)s] " + "w.r.t OpenStack volume [%(stack_vol)s].", + {'cb_vol': cb_volume_id, 'stack_vol': source_volume_id}) + + # Delete volume at CloudByte + if cb_volume_id is not None: + + cb_volumes = self._api_request_for_cloudbyte( + 'listFileSystem', params) + + # Search cb_volume_id in CloudByte volumes + # incase it has already been deleted from CloudByte + cb_volume_id = self._search_volume_id(cb_volumes, cb_volume_id) + + # Delete volume at CloudByte + if cb_volume_id is not None: + + params = {"id": cb_volume_id} + self._api_request_for_cloudbyte('deleteFileSystem', params) + + LOG.info( + _LI("Successfully deleted volume [%(cb_vol)s] " + "at CloudByte corresponding to " + "OpenStack volume [%(stack_vol)s]."), + {'cb_vol': cb_volume_id, + 'stack_vol': source_volume_id}) + + else: + LOG.error(_LE("CloudByte does not have a volume corresponding " + "to OpenStack volume [%s]."), source_volume_id) + + else: + LOG.error(_LE("CloudByte volume information not available for" + " OpenStack volume [%s]."), source_volume_id) + + def create_snapshot(self, snapshot): + """Creates a snapshot at CloudByte.""" + + # OpenStack volume + source_volume_id = snapshot['volume_id'] + + # CloudByte volume id equals OpenStack volume's provider_id + cb_volume_id = snapshot.get('volume').get('provider_id') + + if cb_volume_id is not None: + + snapshot_name = snapshot['display_name'] + if snapshot_name is None or snapshot_name == '': + # Generate the snapshot name + snapshot_name = self._generate_snapshot_name() + # Update the snapshot dict for later use + snapshot['display_name'] = snapshot_name + + params = { + "name": snapshot_name, + "id": cb_volume_id + } + + LOG.debug( + "Will create CloudByte snapshot [%(cb_snap)s] " + "w.r.t CloudByte volume [%(cb_vol)s] " + "and OpenStack volume [%(stack_vol)s].", + {'cb_snap': snapshot_name, + 'cb_vol': cb_volume_id, + 'stack_vol': source_volume_id}) + + self._api_request_for_cloudbyte('createStorageSnapshot', params) + + # Get the snapshot path from CloudByte + path = self._get_cb_snapshot_path(snapshot, cb_volume_id) + + LOG.info( + _LI("Created CloudByte snapshot [%(cb_snap)s] " + "w.r.t CloudByte volume [%(cb_vol)s] " + "and OpenStack volume [%(stack_vol)s]."), + {'cb_snap': path, + 'cb_vol': cb_volume_id, + 'stack_vol': source_volume_id}) + + model_update = {} + # Store snapshot path as snapshot provider_id + model_update['provider_id'] = path + + else: + msg = _("Failed to create snapshot. CloudByte volume information " + "not found for OpenStack volume [%s].") % source_volume_id + raise exception.VolumeBackendAPIException(data=msg) + + return model_update + + def create_cloned_volume(self, cloned_volume, src_volume): + """Create a clone of an existing volume. + + First it will create a snapshot of the source/parent volume, + then it creates a clone of this newly created snapshot. + """ + + # Extract necessary information from input params + parent_volume_id = cloned_volume.get('source_volid') + + # Generating name and id for snapshot + # as this is not user entered in this particular usecase + snapshot_name = self._generate_snapshot_name() + + snapshot_id = (six.text_type(parent_volume_id) + "_" + + time.strftime("%d%m%Y") + time.strftime("%H%M%S")) + + # Prepare the params for create_snapshot + # as well as create_volume_from_snapshot method + snapshot_params = { + 'id': snapshot_id, + 'display_name': snapshot_name, + 'volume_id': parent_volume_id, + 'volume': src_volume, + } + + # Create a snapshot + snapshot = self.create_snapshot(snapshot_params) + snapshot_params['provider_id'] = snapshot.get('provider_id') + + # Create a clone of above snapshot + return self.create_volume_from_snapshot(cloned_volume, snapshot_params) + + def create_volume_from_snapshot(self, cloned_volume, snapshot): + """Create a clone from an existing snapshot.""" + + # Getting necessary data from input params + parent_volume_id = snapshot['volume_id'] + cloned_volume_name = cloned_volume['id'].replace("-", "") + + # CloudByte volume id equals OpenStack volume's provider_id + cb_volume_id = snapshot.get('volume').get('provider_id') + + # CloudByte snapshot path equals OpenStack snapshot's provider_id + cb_snapshot_path = snapshot['provider_id'] + + params = { + "id": cb_volume_id, + "clonename": cloned_volume_name, + "path": cb_snapshot_path + } + + LOG.debug( + "Will create CloudByte clone [%(cb_clone)s] " + "at CloudByte snapshot path [%(cb_snap)s] " + "w.r.t parent OpenStack volume [%(stack_vol)s].", + {'cb_clone': cloned_volume_name, + 'cb_snap': cb_snapshot_path, + 'stack_vol': parent_volume_id}) + + # Create clone of the snapshot + clone_dataset_snapshot_res = ( + self._api_request_for_cloudbyte('cloneDatasetSnapshot', params)) + + cb_snap = clone_dataset_snapshot_res.get('cloneDatasetSnapshot') + + cb_vol = {} + if cb_snap is not None: + cb_vol = cb_snap.get('filesystem') + else: + msg = ("Error: Clone creation failed for " + "OpenStack volume [%(vol)s] with CloudByte " + "snapshot path [%(path)s]" % + {'vol': parent_volume_id, 'path': cb_snapshot_path}) + raise exception.VolumeBackendAPIException(data=msg) + + LOG.info( + _LI("Created a clone [%(cb_clone)s] " + "at CloudByte snapshot path [%(cb_snap)s] " + "w.r.t parent OpenStack volume [%(stack_vol)s]."), + {'cb_clone': cloned_volume_name, + 'cb_snap': cb_snapshot_path, + 'stack_vol': parent_volume_id}) + + return self._build_provider_details_from_volume(cb_vol) + + def delete_snapshot(self, snapshot): + """Delete a snapshot at CloudByte.""" + + # Find volume id + source_volume_id = snapshot['volume_id'] + + # CloudByte volume id equals OpenStack volume's provider_id + cb_volume_id = snapshot.get('volume').get('provider_id') + + # CloudByte snapshot path equals OpenStack snapshot's provider_id + cb_snapshot_path = snapshot['provider_id'] + + # If cb_snapshot_path is 'None' + # then no need to execute CloudByte API + if cb_snapshot_path is not None: + + params = { + "id": cb_volume_id, + "path": cb_snapshot_path + } + + LOG.debug("Will delete CloudByte snapshot [%(snap)s] w.r.t " + "parent CloudByte volume [%(cb_vol)s] " + "and parent OpenStack volume [%(stack_vol)s].", + {'snap': cb_snapshot_path, + 'cb_vol': cb_volume_id, + 'stack_vol': source_volume_id}) + + # Execute CloudByte API + self._api_request_for_cloudbyte('deleteSnapshot', params) + LOG.info( + _LI("Deleted CloudByte snapshot [%(snap)s] w.r.t " + "parent CloudByte volume [%(cb_vol)s] " + "and parent OpenStack volume [%(stack_vol)s]."), + {'snap': cb_snapshot_path, + 'cb_vol': cb_volume_id, + 'stack_vol': source_volume_id}) + + else: + LOG.error(_LE("CloudByte snapshot information is not available" + " for OpenStack volume [%s]."), source_volume_id) + + def extend_volume(self, volume, new_size): + + # CloudByte volume id equals OpenStack volume's provider_id + cb_volume_id = volume.get('provider_id') + + params = { + "id": cb_volume_id, + "quotasize": six.text_type(new_size) + 'G' + } + + # Request the CloudByte api to update the volume + self._api_request_for_cloudbyte('updateFileSystem', params) + + def create_export(self, context, volume): + """Setup the iscsi export info.""" + model_update = {} + + # Will provide CHAP Authentication on forthcoming patches/release + model_update['provider_auth'] = None + + return model_update + + def ensure_export(self, context, volume): + """Verify the iscsi export info.""" + model_update = {} + + # Will provide CHAP Authentication on forthcoming patches/release + model_update['provider_auth'] = None + + return model_update + + def get_volume_stats(self, refresh=False): + """Get volume statistics. + + If 'refresh' is True, update/refresh the statistics first. + """ + + if refresh: + # Get the TSM name from configuration + tsm_name = self.configuration.cb_tsm_name + # Get the storage details of this TSM + data = self._get_storage_info(tsm_name) + + data["volume_backend_name"] = ( + self.configuration.safe_get('volume_backend_name') or + 'CloudByte') + data["vendor_name"] = 'CloudByte' + data['reserved_percentage'] = 0 + data["driver_version"] = CloudByteISCSIDriver.VERSION + data["storage_protocol"] = 'iSCSI' + + LOG.debug("CloudByte driver stats: [%s].", data) + # Set this to the instance variable + self.volume_stats = data + + return self.volume_stats diff --git a/cinder/volume/drivers/cloudbyte/options.py b/cinder/volume/drivers/cloudbyte/options.py new file mode 100644 index 000000000..f140a5377 --- /dev/null +++ b/cinder/volume/drivers/cloudbyte/options.py @@ -0,0 +1,74 @@ +# Copyright 2015 CloudByte Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +cloudbyte_connection_opts = [ + cfg.StrOpt("cb_apikey", + default="None", + help="Driver will use this API key to authenticate " + "against the CloudByte storage's management interface."), + cfg.StrOpt("cb_account_name", + default="None", + help="CloudByte storage specific account name. " + "This maps to a project name in OpenStack."), + cfg.StrOpt("cb_tsm_name", + default="None", + help="This corresponds to the name of " + "Tenant Storage Machine (TSM) in CloudByte storage. " + "A volume will be created in this TSM."), + cfg.IntOpt("cb_confirm_volume_create_retry_interval", + default=5, + help="A retry value in seconds. Will be used by the driver " + "to check if volume creation was successful in " + "CloudByte storage."), + cfg.IntOpt("cb_confirm_volume_create_retries", + default=3, + help="Will confirm a successful volume " + "creation in CloudByte storage by making " + "this many number of attempts."), ] + +cloudbyte_add_qosgroup_opts = [ + cfg.DictOpt('cb_add_qosgroup', + default={ + 'iops': '10', + 'latency': '15', + 'graceallowed': 'false', + 'networkspeed': '0', + 'memlimit': '0', + 'tpcontrol': 'false', + 'throughput': '0', + 'iopscontrol': 'true' + }, + help="These values will be used for CloudByte storage's " + "addQos API call."), ] + +cloudbyte_create_volume_opts = [ + cfg.DictOpt('cb_create_volume', + default={ + 'blocklength': '512B', + 'compression': 'off', + 'deduplication': 'off', + 'sync': 'always', + 'recordsize': '16k', + 'protocoltype': 'ISCSI' + }, + help="These values will be used for CloudByte storage's " + "createVolume API call."), ] + +CONF = cfg.CONF +CONF.register_opts(cloudbyte_add_qosgroup_opts) +CONF.register_opts(cloudbyte_create_volume_opts) +CONF.register_opts(cloudbyte_connection_opts)