From 4f7980840717c2b5d87219dfdb43ad3327d1c288 Mon Sep 17 00:00:00 2001
From: smile-luobin <luobin_smile@163.com>
Date: Sun, 14 Feb 2016 07:13:03 +0000
Subject: [PATCH] Implement swift bank plugin
A basic implement swift bank plugin
Change-Id: I7c809aed46b225e3de5d298d7c1bb650fab7a139
Closes-Bug: #1545384
---
requirements.txt | 1 +
smaug/common/config.py | 10 +
smaug/exception.py | 28 ++
smaug/services/protection/bank_plugin.py | 4 +-
.../protection/bank_plugins/__init__.py | 0
.../bank_plugins/swift_bank_plugin.py | 271 ++++++++++++++++++
.../unit/protection/fake_swift_client.py | 99 +++++++
.../unit/protection/test_swift_bank_plugin.py | 104 +++++++
test-requirements.txt | 1 +
9 files changed, 516 insertions(+), 2 deletions(-)
create mode 100644 smaug/services/protection/bank_plugins/__init__.py
create mode 100644 smaug/services/protection/bank_plugins/swift_bank_plugin.py
create mode 100644 smaug/tests/unit/protection/fake_swift_client.py
create mode 100644 smaug/tests/unit/protection/test_swift_bank_plugin.py
diff --git a/requirements.txt b/requirements.txt
index 5120674d..9151b94d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -29,3 +29,4 @@ sqlalchemy-migrate>=0.9.6
stevedore>=1.5.0 # Apache-2.0
WebOb>=1.2.3
oslo.i18n>=1.5.0 # Apache-2.0
+python-swiftclient>=2.2.0 # Apache-2.0
diff --git a/smaug/common/config.py b/smaug/common/config.py
index f17ab9d4..01e5876d 100644
--- a/smaug/common/config.py
+++ b/smaug/common/config.py
@@ -70,6 +70,16 @@ global_opts = [
choices=['noauth', 'keystone'],
help='The strategy to use for auth. Supports noauth or '
'keystone.'),
+ cfg.IntOpt('lease_renew_window',
+ default=120,
+ help='period for bank lease, in seconds, '
+ 'between bank lease client renew the lease'),
+ cfg.IntOpt('lease_expire_window',
+ default=600,
+ help='expired_window for bank lease, in seconds'),
+ cfg.IntOpt('lease_validity_window',
+ default=100,
+ help='validity_window for bank lease, in seconds'),
]
CONF.register_opts(global_opts)
diff --git a/smaug/exception.py b/smaug/exception.py
index c8065906..b3c894aa 100644
--- a/smaug/exception.py
+++ b/smaug/exception.py
@@ -233,3 +233,31 @@ class ProviderNotFound(NotFound):
class CheckpointNotFound(NotFound):
message = _("Checkpoint %(checkpoint_id)s could"
" not be found.")
+
+
+class BankCreateObjectFailed(SmaugException):
+ message = _("Create Object in Bank Failed: %(reason)s")
+
+
+class BankUpdateObjectFailed(SmaugException):
+ message = _("Update Object %(key)s in Bank Failed: %(reason)s")
+
+
+class BankDeleteObjectFailed(SmaugException):
+ message = _("Delete Object %(key)s in Bank Failed: %(reason)s")
+
+
+class BankGetObjectFailed(SmaugException):
+ message = _("Get Object %(key)s in Bank Failed: %(reason)s")
+
+
+class BankListObjectsFailed(SmaugException):
+ message = _("Get Object in Bank Failed: %(reason)s")
+
+
+class AcquireLeaseFailed(SmaugException):
+ message = _("Acquire Lease in Failed: %(reason)s")
+
+
+class CreateContainerFailed(SmaugException):
+ message = _("Create Container in Bank Failed: %(reason)s")
diff --git a/smaug/services/protection/bank_plugin.py b/smaug/services/protection/bank_plugin.py
index 66b40866..e4b6be7b 100644
--- a/smaug/services/protection/bank_plugin.py
+++ b/smaug/services/protection/bank_plugin.py
@@ -28,12 +28,12 @@ LOG = logging.getLogger(__name__)
@six.add_metaclass(abc.ABCMeta)
class LeasePlugin(object):
@abc.abstractmethod
- def acquire_lease(self, owner_id):
+ def acquire_lease(self):
# TODO(wangliuan)
pass
@abc.abstractmethod
- def renew_lease(self, owner_id):
+ def renew_lease(self):
# TODO(wangliuan)
pass
diff --git a/smaug/services/protection/bank_plugins/__init__.py b/smaug/services/protection/bank_plugins/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/smaug/services/protection/bank_plugins/swift_bank_plugin.py b/smaug/services/protection/bank_plugins/swift_bank_plugin.py
new file mode 100644
index 00000000..3b7e5aac
--- /dev/null
+++ b/smaug/services/protection/bank_plugins/swift_bank_plugin.py
@@ -0,0 +1,271 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_service import loopingcall
+from smaug import exception
+from smaug.i18n import _, _LE
+from smaug.services.protection.bank_plugin import BankPlugin
+from smaug.services.protection.bank_plugin import LeasePlugin
+from swiftclient import client as swift
+from swiftclient import ClientException
+import time
+import uuid
+
+swift_client_opts = [
+ cfg.StrOpt('bank_swift_url',
+ help='The URL of the Swift endpoint'),
+ cfg.StrOpt('bank_swift_auth_url',
+ help='The URL of the Keystone endpoint'),
+ cfg.StrOpt('bank_swift_auth',
+ default='single_user',
+ help='Swift authentication mechanism'),
+ cfg.StrOpt('bank_swift_auth_version',
+ default='1',
+ help='Swift authentication version. '
+ 'Specify "1" for auth 1.0, or "2" for auth 2.0'),
+ cfg.StrOpt('bank_swift_tenant_name',
+ help='Swift tenant/account name. '
+ 'Required when connecting to an auth 2.0 system'),
+ cfg.StrOpt('bank_swift_user',
+ help='Swift user name'),
+ cfg.StrOpt('bank_swift_key',
+ help='Swift key for authentication'),
+ cfg.IntOpt('bank_swift_retry_attempts',
+ default=3,
+ help='The number of retries to make for '
+ 'Swift operations'),
+ cfg.IntOpt('bank_swift_retry_backoff',
+ default=2,
+ help='The backoff time in seconds '
+ 'between Swift retries'),
+ cfg.StrOpt('bank_swift_ca_cert_file',
+ help='Location of the CA certificate file '
+ 'to use for swift client requests.'),
+ cfg.BoolOpt('bank_swift_auth_insecure',
+ default=False,
+ help='Bypass verification of server certificate when '
+ 'making SSL connection to Swift.'),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(swift_client_opts, "swift_client")
+LOG = logging.getLogger(__name__)
+
+
+class SwiftConnectionFailed(exception.SmaugException):
+ message = _("Connection to swift failed: %(reason)s")
+
+
+class SwiftBankPlugin(BankPlugin, LeasePlugin):
+ def __init__(self, context, object_container):
+ super(BankPlugin, self).__init__()
+ self.context = context
+ self.swift_retry_attempts = CONF.swift_client.bank_swift_retry_attempts
+ self.swift_retry_backoff = CONF.swift_client.bank_swift_retry_backoff
+ self.swift_auth_insecure = CONF.swift_client.bank_swift_auth_insecure
+ self.swift_ca_cert_file = CONF.swift_client.bank_swift_ca_cert_file
+ self.lease_expire_window = CONF.lease_expire_window
+ self.lease_renew_window = CONF.lease_renew_window
+ # TODO(luobin):
+ # init lease_validity_window
+ # according to lease_renew_window if not configured
+ self.lease_validity_window = CONF.lease_validity_window
+
+ # TODO(luobin): create a uuid of this bank_plugin
+ self.owner_id = str(uuid.uuid4())
+ self.lease_expire_time = 0
+ self.bank_leases_container = "leases"
+ self.bank_object_container = object_container
+ self.connection = self._setup_connection()
+
+ # create container
+ try:
+ self._put_container(self.bank_object_container)
+ self._put_container(self.bank_leases_container)
+ except SwiftConnectionFailed as err:
+ LOG.error(_LE("bank plugin create container failed."))
+ raise exception.CreateContainerFailed(reason=err)
+
+ # acquire lease
+ try:
+ self.acquire_lease()
+ except exception.AcquireLeaseFailed as err:
+ LOG.error(_LE("bank plugin acquire lease failed."))
+ raise err
+
+ # start renew lease
+ renew_lease_loop = loopingcall.FixedIntervalLoopingCall(
+ self.renew_lease)
+ renew_lease_loop.start(interval=self.lease_renew_window,
+ initial_delay=self.lease_renew_window)
+
+ def _setup_connection(self):
+ if CONF.swift_client.bank_swift_auth == "single_user":
+ connection = swift.Connection(
+ authurl=CONF.swift_client.bank_swift_auth_url,
+ auth_version=CONF.swift_client.bank_swift_auth_version,
+ tenant_name=CONF.swift_client.bank_swift_tenant_name,
+ user=CONF.swift_client.bank_swift_user,
+ key=CONF.swift_client.bank_swift_key,
+ retries=self.swift_retry_attempts,
+ starting_backoff=self.swift_retry_backoff,
+ insecure=self.swift_auth_insecure,
+ cacert=self.swift_ca_cert_file)
+ else:
+ connection = swift.Connection(
+ preauthurl=CONF.swift_client.bank_swift_url,
+ preauthtoken=self.context.auth_token,
+ retries=self.swift_retry_attempts,
+ starting_backoff=self.swift_retry_backoff,
+ insecure=self.swift_auth_insecure,
+ cacert=self.swift_ca_cert_file)
+ return connection
+
+ def create_object(self, key, value):
+ try:
+ self._put_object(container=self.bank_object_container,
+ obj=key,
+ contents=value)
+ except SwiftConnectionFailed as err:
+ LOG.error(_LE("create object failed, err: %s."), err)
+ raise exception.BankCreateObjectFailed(reasone=err,
+ key=key)
+
+ def update_object(self, key, value):
+ try:
+ self._put_object(container=self.bank_object_container,
+ obj=key,
+ contents=value)
+ except SwiftConnectionFailed as err:
+ LOG.error(_LE("update object failed, err: %s."), err)
+ raise exception.BankUpdateObjectFailed(reasone=err,
+ key=key)
+
+ def delete_object(self, key):
+ try:
+ self._delete_object(container=self.bank_object_container,
+ obj=key)
+ except SwiftConnectionFailed as err:
+ LOG.error(_LE("delete object failed, err: %s."), err)
+ raise exception.BankDeleteObjectFailed(reasone=err,
+ key=key)
+
+ def get_object(self, key):
+ try:
+ return self._get_object(container=self.bank_object_container,
+ obj=key)
+ except SwiftConnectionFailed as err:
+ LOG.error(_LE("get object failed, err: %s."), err)
+ raise exception.BankGetObjectFailed(reasone=err,
+ key=key)
+
+ def list_objects(self, prefix=None, limit=None, marker=None):
+ object_names = []
+ try:
+ body = self._get_container(container=self.bank_object_container,
+ prefix=prefix,
+ limit=limit,
+ marker=marker)
+ except SwiftConnectionFailed as err:
+ LOG.error(_LE("list objects failed, err: %s."), err)
+ raise exception.BankListObjectsFailed(reasone=err)
+ for obj in body:
+ if obj.get("name"):
+ object_names.append(obj.get("name"))
+ return object_names
+
+ def acquire_lease(self):
+ container = self.bank_leases_container
+ obj = self.owner_id
+ contents = self.owner_id
+ headers = {'X-Delete-After': self.lease_expire_window}
+ try:
+ self._put_object(container=container,
+ obj=obj,
+ contents=contents,
+ headers=headers)
+ self.lease_expire_time = long(
+ time.time()) + self.lease_expire_window
+ except SwiftConnectionFailed as err:
+ LOG.error(_LE("acquire lease failed, err:%s."), err)
+ raise exception.AcquireLeaseFailed(reason=err)
+
+ def renew_lease(self):
+ container = self.bank_leases_container
+ obj = self.owner_id
+ headers = {'X-Delete-After': self.lease_expire_window}
+ try:
+ self._post_object(container=container,
+ obj=obj,
+ headers=headers)
+ self.lease_expire_time = long(
+ time.time()) + self.lease_expire_window
+ except SwiftConnectionFailed as err:
+ LOG.error(_LE("acquire lease failed, err:%s."), err)
+
+ def check_lease_validity(self):
+ if (self.lease_expire_time - long(time.time()) >=
+ self.lease_validity_window):
+ return True
+ else:
+ return False
+
+ def _put_object(self, container, obj, contents, headers=None):
+ try:
+ self.connection.put_object(container=container,
+ obj=obj,
+ contents=contents,
+ headers=headers)
+ except ClientException as err:
+ raise SwiftConnectionFailed(reason=err)
+
+ def _get_object(self, container, obj):
+ try:
+ (_resp, body) = self.connection.get_object(container=container,
+ obj=obj)
+ return body
+ except ClientException as err:
+ raise SwiftConnectionFailed(reason=err)
+
+ def _post_object(self, container, obj, headers):
+ try:
+ self.connection.post_object(container=container,
+ obj=obj,
+ headers=headers)
+ except ClientException as err:
+ raise SwiftConnectionFailed(reason=err)
+
+ def _delete_object(self, container, obj):
+ try:
+ self.connection.delete_object(container=container,
+ obj=obj)
+ except ClientException as err:
+ raise SwiftConnectionFailed(reason=err)
+
+ def _put_container(self, container):
+ try:
+ self.connection.put_container(container=container)
+ except ClientException as err:
+ raise SwiftConnectionFailed(reason=err)
+
+ def _get_container(self, container, prefix=None, limit=None, marker=None):
+ try:
+ (_resp, body) = self.connection.get_container(
+ container=container,
+ prefix=prefix,
+ limit=limit,
+ marker=marker)
+ return body
+ except ClientException as err:
+ raise SwiftConnectionFailed(reason=err)
diff --git a/smaug/tests/unit/protection/fake_swift_client.py b/smaug/tests/unit/protection/fake_swift_client.py
new file mode 100644
index 00000000..8cc332bb
--- /dev/null
+++ b/smaug/tests/unit/protection/fake_swift_client.py
@@ -0,0 +1,99 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+from swiftclient import ClientException
+import tempfile
+
+
+class FakeSwiftClient(object):
+ def __init__(self, *args, **kwargs):
+ pass
+
+ @classmethod
+ def connection(cls, *args, **kargs):
+ return FakeSwiftConnection()
+
+
+class FakeSwiftConnection(object):
+ def __init__(self, *args, **kwargs):
+ self.swiftdir = tempfile.mkdtemp()
+
+ def put_container(self, container):
+ container_dir = self.swiftdir + "/" + container
+ if os.path.exists(container_dir) is True:
+ return
+ else:
+ os.makedirs(container_dir)
+
+ def get_container(self, container, prefix, limit, marker):
+ container_dir = self.swiftdir + "/" + container
+ body = []
+ if prefix:
+ objects_dir = container_dir + "/" + prefix
+ else:
+ objects_dir = container_dir
+ for f in os.listdir(objects_dir):
+ if os.path.isfile(objects_dir + "/" + f):
+ body.append({"name": f})
+ else:
+ body.append({"subdir": f})
+ return None, body
+
+ def put_object(self, container, obj, contents, headers=None):
+ container_dir = self.swiftdir + "/" + container
+ obj_file = container_dir + "/" + obj
+ obj_dir = obj_file[0:obj_file.rfind("/")]
+ if os.path.exists(container_dir) is True:
+ if os.path.exists(obj_dir) is False:
+ os.makedirs(obj_dir)
+ with open(obj_file, "w") as f:
+ f.write(contents)
+ return
+ else:
+ raise ClientException("error_container")
+
+ def get_object(self, container, obj):
+ container_dir = self.swiftdir + "/" + container
+ obj_file = container_dir + "/" + obj
+ if os.path.exists(container_dir) is True:
+ if os.path.exists(obj_file) is True:
+ with open(obj_file, "r") as f:
+ return None, f.read()
+ else:
+ raise ClientException("error_obj")
+ else:
+ raise ClientException("error_container")
+
+ def delete_object(self, container, obj):
+ container_dir = self.swiftdir + "/" + container
+ obj_file = container_dir + "/" + obj
+ if os.path.exists(container_dir) is True:
+ if os.path.exists(obj_file) is True:
+ os.remove(obj_file)
+ else:
+ raise ClientException("error_obj")
+ else:
+ raise ClientException("error_container")
+
+ def update_object(self, container, obj, contents):
+ container_dir = self.swiftdir + "/" + container
+ obj_file = container_dir + "/" + obj
+ if os.path.exists(container_dir) is True:
+ if os.path.exists(obj_file) is True:
+ with open(obj_file, "w") as f:
+ f.write(contents)
+ return
+ else:
+ raise ClientException("error_obj")
+ else:
+ raise ClientException("error_container")
diff --git a/smaug/tests/unit/protection/test_swift_bank_plugin.py b/smaug/tests/unit/protection/test_swift_bank_plugin.py
new file mode 100644
index 00000000..6e7a0c98
--- /dev/null
+++ b/smaug/tests/unit/protection/test_swift_bank_plugin.py
@@ -0,0 +1,104 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+import os
+from oslo_config import cfg
+from oslo_utils import importutils
+from smaug.tests import base
+from smaug.tests.unit.protection.fake_swift_client import FakeSwiftClient
+from swiftclient import client as swift
+import time
+
+CONF = cfg.CONF
+
+
+class FakeConf(object):
+ def __init__(self):
+ self.lease_expire_window = 600
+ self.lease_renew_window = 120
+ self.lease_validity_window = 100
+
+
+class SwiftBankPluginTest(base.TestCase):
+ def setUp(self):
+ super(SwiftBankPluginTest, self).setUp()
+ self.conf = FakeConf()
+ self.fake_connection = FakeSwiftClient.connection()
+ import_str = "smaug.services.protection.bank_plugins." \
+ "swift_bank_plugin.SwiftBankPlugin"
+ self.object_container = "objects"
+ swift_bank_plugin_cls = importutils.import_class(
+ import_str=import_str)
+ swift.Connection = mock.MagicMock()
+ swift.Connection.return_value = self.fake_connection
+ self.swift_bank_plugin = swift_bank_plugin_cls(None,
+ self.object_container)
+
+ def test_acquire_lease(self):
+ self.swift_bank_plugin.acquire_lease()
+ expire_time = long(time.time()) + self.conf.lease_expire_window
+ self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
+
+ def test_renew_lease(self):
+ self.swift_bank_plugin.acquire_lease()
+ expire_time = long(time.time()) + self.conf.lease_expire_window
+ self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
+ time.sleep(5)
+ self.swift_bank_plugin.acquire_lease()
+ expire_time = long(time.time()) + self.conf.lease_expire_window
+ self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
+
+ def test_check_lease_validity(self):
+ self.swift_bank_plugin.acquire_lease()
+ expire_time = long(time.time()) + self.conf.lease_expire_window
+ self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
+ is_valid = self.swift_bank_plugin.check_lease_validity()
+ self.assertEqual(is_valid, True)
+
+ def test_create_object(self):
+ self.swift_bank_plugin.create_object("key-1", "value-1")
+ object_file = os.path.join(self.fake_connection.swiftdir,
+ self.object_container,
+ "key-1")
+ with open(object_file, "r") as f:
+ contents = f.read()
+ self.assertEqual(contents, "value-1")
+
+ def test_delete_object(self):
+ self.swift_bank_plugin.create_object("key", "value")
+ self.swift_bank_plugin.delete_object("key")
+ object_file = os.path.join(self.fake_connection.swiftdir,
+ self.object_container,
+ "key")
+ self.assertEqual(os.path.isfile(object_file), False)
+
+ def test_get_object(self):
+ self.swift_bank_plugin.create_object("key", "value")
+ value = self.swift_bank_plugin.get_object("key")
+ self.assertEqual(value, "value")
+
+ def test_list_objects(self):
+ self.swift_bank_plugin.create_object("key-1", "value-1")
+ self.swift_bank_plugin.create_object("key-2", "value-2")
+ objects = self.swift_bank_plugin.list_objects(prefix=None)
+ self.assertEqual(len(objects), 2)
+
+ def test_update_object(self):
+ self.swift_bank_plugin.create_object("key-1", "value-1")
+ self.swift_bank_plugin.update_object("key-1", "value-2")
+ object_file = os.path.join(self.fake_connection.swiftdir,
+ self.object_container,
+ "key-1")
+ with open(object_file, "r") as f:
+ contents = f.read()
+ self.assertEqual(contents, "value-2")
diff --git a/test-requirements.txt b/test-requirements.txt
index 0ecbfef8..78e691a9 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -13,3 +13,4 @@ oslotest>=1.10.0 # Apache-2.0
testrepository>=0.0.18
testscenarios>=0.4
testtools>=1.4.0
+python-swiftclient>=2.2.0 # Apache-2.0