Implement swift bank plugin

A basic implement swift bank plugin

Change-Id: I7c809aed46b225e3de5d298d7c1bb650fab7a139
Closes-Bug: #1545384

requirements.txt

@@ -29,3 +29,4 @@ sqlalchemy-migrate>=0.9.6
 stevedore>=1.5.0 # Apache-2.0
 WebOb>=1.2.3
 oslo.i18n>=1.5.0 # Apache-2.0
+python-swiftclient>=2.2.0 # Apache-2.0

smaug/common/config.py

@@ -70,6 +70,16 @@ global_opts = [
                choices=['noauth', 'keystone'],
                help='The strategy to use for auth. Supports noauth or '
                     'keystone.'),
+    cfg.IntOpt('lease_renew_window',
+               default=120,
+               help='period for bank lease, in seconds, '
+                    'between bank lease client renew the lease'),
+    cfg.IntOpt('lease_expire_window',
+               default=600,
+               help='expired_window for bank lease, in seconds'),
+    cfg.IntOpt('lease_validity_window',
+               default=100,
+               help='validity_window for bank lease, in seconds'),
 ]
 
 CONF.register_opts(global_opts)

smaug/exception.py

@@ -233,3 +233,31 @@ class ProviderNotFound(NotFound):
 class CheckpointNotFound(NotFound):
     message = _("Checkpoint %(checkpoint_id)s could"
                 " not be found.")
+
+
+class BankCreateObjectFailed(SmaugException):
+    message = _("Create Object in Bank Failed: %(reason)s")
+
+
+class BankUpdateObjectFailed(SmaugException):
+    message = _("Update Object %(key)s in Bank Failed: %(reason)s")
+
+
+class BankDeleteObjectFailed(SmaugException):
+    message = _("Delete Object %(key)s in Bank Failed: %(reason)s")
+
+
+class BankGetObjectFailed(SmaugException):
+    message = _("Get Object %(key)s in Bank Failed: %(reason)s")
+
+
+class BankListObjectsFailed(SmaugException):
+    message = _("Get Object in Bank Failed: %(reason)s")
+
+
+class AcquireLeaseFailed(SmaugException):
+    message = _("Acquire Lease in Failed: %(reason)s")
+
+
+class CreateContainerFailed(SmaugException):
+    message = _("Create Container in Bank Failed: %(reason)s")

smaug/services/protection/bank_plugin.py

@@ -28,12 +28,12 @@ LOG = logging.getLogger(__name__)
 @six.add_metaclass(abc.ABCMeta)
 class LeasePlugin(object):
     @abc.abstractmethod
-    def acquire_lease(self, owner_id):
+    def acquire_lease(self):
         # TODO(wangliuan)
         pass
 
     @abc.abstractmethod
-    def renew_lease(self, owner_id):
+    def renew_lease(self):
         # TODO(wangliuan)
         pass
 

smaug/services/protection/bank_plugins/swift_bank_plugin.py

@@ -0,0 +1,271 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_log import log as logging
+from oslo_service import loopingcall
+from smaug import exception
+from smaug.i18n import _, _LE
+from smaug.services.protection.bank_plugin import BankPlugin
+from smaug.services.protection.bank_plugin import LeasePlugin
+from swiftclient import client as swift
+from swiftclient import ClientException
+import time
+import uuid
+
+swift_client_opts = [
+    cfg.StrOpt('bank_swift_url',
+               help='The URL of the Swift endpoint'),
+    cfg.StrOpt('bank_swift_auth_url',
+               help='The URL of the Keystone endpoint'),
+    cfg.StrOpt('bank_swift_auth',
+               default='single_user',
+               help='Swift authentication mechanism'),
+    cfg.StrOpt('bank_swift_auth_version',
+               default='1',
+               help='Swift authentication version. '
+                    'Specify "1" for auth 1.0, or "2" for auth 2.0'),
+    cfg.StrOpt('bank_swift_tenant_name',
+               help='Swift tenant/account name. '
+                    'Required when connecting to an auth 2.0 system'),
+    cfg.StrOpt('bank_swift_user',
+               help='Swift user name'),
+    cfg.StrOpt('bank_swift_key',
+               help='Swift key for authentication'),
+    cfg.IntOpt('bank_swift_retry_attempts',
+               default=3,
+               help='The number of retries to make for '
+                    'Swift operations'),
+    cfg.IntOpt('bank_swift_retry_backoff',
+               default=2,
+               help='The backoff time in seconds '
+                    'between Swift retries'),
+    cfg.StrOpt('bank_swift_ca_cert_file',
+               help='Location of the CA certificate file '
+                    'to use for swift client requests.'),
+    cfg.BoolOpt('bank_swift_auth_insecure',
+                default=False,
+                help='Bypass verification of server certificate when '
+                     'making SSL connection to Swift.'),
+]
+
+CONF = cfg.CONF
+CONF.register_opts(swift_client_opts, "swift_client")
+LOG = logging.getLogger(__name__)
+
+
+class SwiftConnectionFailed(exception.SmaugException):
+    message = _("Connection to swift failed: %(reason)s")
+
+
+class SwiftBankPlugin(BankPlugin, LeasePlugin):
+    def __init__(self, context, object_container):
+        super(BankPlugin, self).__init__()
+        self.context = context
+        self.swift_retry_attempts = CONF.swift_client.bank_swift_retry_attempts
+        self.swift_retry_backoff = CONF.swift_client.bank_swift_retry_backoff
+        self.swift_auth_insecure = CONF.swift_client.bank_swift_auth_insecure
+        self.swift_ca_cert_file = CONF.swift_client.bank_swift_ca_cert_file
+        self.lease_expire_window = CONF.lease_expire_window
+        self.lease_renew_window = CONF.lease_renew_window
+        # TODO(luobin):
+        # init lease_validity_window
+        # according to lease_renew_window if not configured
+        self.lease_validity_window = CONF.lease_validity_window
+
+        # TODO(luobin): create a uuid of this bank_plugin
+        self.owner_id = str(uuid.uuid4())
+        self.lease_expire_time = 0
+        self.bank_leases_container = "leases"
+        self.bank_object_container = object_container
+        self.connection = self._setup_connection()
+
+        # create container
+        try:
+            self._put_container(self.bank_object_container)
+            self._put_container(self.bank_leases_container)
+        except SwiftConnectionFailed as err:
+            LOG.error(_LE("bank plugin create container failed."))
+            raise exception.CreateContainerFailed(reason=err)
+
+        # acquire lease
+        try:
+            self.acquire_lease()
+        except exception.AcquireLeaseFailed as err:
+            LOG.error(_LE("bank plugin acquire lease failed."))
+            raise err
+
+        # start renew lease
+        renew_lease_loop = loopingcall.FixedIntervalLoopingCall(
+            self.renew_lease)
+        renew_lease_loop.start(interval=self.lease_renew_window,
+                               initial_delay=self.lease_renew_window)
+
+    def _setup_connection(self):
+        if CONF.swift_client.bank_swift_auth == "single_user":
+            connection = swift.Connection(
+                authurl=CONF.swift_client.bank_swift_auth_url,
+                auth_version=CONF.swift_client.bank_swift_auth_version,
+                tenant_name=CONF.swift_client.bank_swift_tenant_name,
+                user=CONF.swift_client.bank_swift_user,
+                key=CONF.swift_client.bank_swift_key,
+                retries=self.swift_retry_attempts,
+                starting_backoff=self.swift_retry_backoff,
+                insecure=self.swift_auth_insecure,
+                cacert=self.swift_ca_cert_file)
+        else:
+            connection = swift.Connection(
+                preauthurl=CONF.swift_client.bank_swift_url,
+                preauthtoken=self.context.auth_token,
+                retries=self.swift_retry_attempts,
+                starting_backoff=self.swift_retry_backoff,
+                insecure=self.swift_auth_insecure,
+                cacert=self.swift_ca_cert_file)
+        return connection
+
+    def create_object(self, key, value):
+        try:
+            self._put_object(container=self.bank_object_container,
+                             obj=key,
+                             contents=value)
+        except SwiftConnectionFailed as err:
+            LOG.error(_LE("create object failed, err: %s."), err)
+            raise exception.BankCreateObjectFailed(reasone=err,
+                                                   key=key)
+
+    def update_object(self, key, value):
+        try:
+            self._put_object(container=self.bank_object_container,
+                             obj=key,
+                             contents=value)
+        except SwiftConnectionFailed as err:
+            LOG.error(_LE("update object failed, err: %s."), err)
+            raise exception.BankUpdateObjectFailed(reasone=err,
+                                                   key=key)
+
+    def delete_object(self, key):
+        try:
+            self._delete_object(container=self.bank_object_container,
+                                obj=key)
+        except SwiftConnectionFailed as err:
+            LOG.error(_LE("delete object failed, err: %s."), err)
+            raise exception.BankDeleteObjectFailed(reasone=err,
+                                                   key=key)
+
+    def get_object(self, key):
+        try:
+            return self._get_object(container=self.bank_object_container,
+                                    obj=key)
+        except SwiftConnectionFailed as err:
+            LOG.error(_LE("get object failed, err: %s."), err)
+            raise exception.BankGetObjectFailed(reasone=err,
+                                                key=key)
+
+    def list_objects(self, prefix=None, limit=None, marker=None):
+        object_names = []
+        try:
+            body = self._get_container(container=self.bank_object_container,
+                                       prefix=prefix,
+                                       limit=limit,
+                                       marker=marker)
+        except SwiftConnectionFailed as err:
+            LOG.error(_LE("list objects failed, err: %s."), err)
+            raise exception.BankListObjectsFailed(reasone=err)
+        for obj in body:
+            if obj.get("name"):
+                object_names.append(obj.get("name"))
+        return object_names
+
+    def acquire_lease(self):
+        container = self.bank_leases_container
+        obj = self.owner_id
+        contents = self.owner_id
+        headers = {'X-Delete-After': self.lease_expire_window}
+        try:
+            self._put_object(container=container,
+                             obj=obj,
+                             contents=contents,
+                             headers=headers)
+            self.lease_expire_time = long(
+                time.time()) + self.lease_expire_window
+        except SwiftConnectionFailed as err:
+            LOG.error(_LE("acquire lease failed, err:%s."), err)
+            raise exception.AcquireLeaseFailed(reason=err)
+
+    def renew_lease(self):
+        container = self.bank_leases_container
+        obj = self.owner_id
+        headers = {'X-Delete-After': self.lease_expire_window}
+        try:
+            self._post_object(container=container,
+                              obj=obj,
+                              headers=headers)
+            self.lease_expire_time = long(
+                time.time()) + self.lease_expire_window
+        except SwiftConnectionFailed as err:
+            LOG.error(_LE("acquire lease failed, err:%s."), err)
+
+    def check_lease_validity(self):
+        if (self.lease_expire_time - long(time.time()) >=
+                self.lease_validity_window):
+            return True
+        else:
+            return False
+
+    def _put_object(self, container, obj, contents, headers=None):
+        try:
+            self.connection.put_object(container=container,
+                                       obj=obj,
+                                       contents=contents,
+                                       headers=headers)
+        except ClientException as err:
+            raise SwiftConnectionFailed(reason=err)
+
+    def _get_object(self, container, obj):
+        try:
+            (_resp, body) = self.connection.get_object(container=container,
+                                                       obj=obj)
+            return body
+        except ClientException as err:
+            raise SwiftConnectionFailed(reason=err)
+
+    def _post_object(self, container, obj, headers):
+        try:
+            self.connection.post_object(container=container,
+                                        obj=obj,
+                                        headers=headers)
+        except ClientException as err:
+            raise SwiftConnectionFailed(reason=err)
+
+    def _delete_object(self, container, obj):
+        try:
+            self.connection.delete_object(container=container,
+                                          obj=obj)
+        except ClientException as err:
+            raise SwiftConnectionFailed(reason=err)
+
+    def _put_container(self, container):
+        try:
+            self.connection.put_container(container=container)
+        except ClientException as err:
+            raise SwiftConnectionFailed(reason=err)
+
+    def _get_container(self, container, prefix=None, limit=None, marker=None):
+        try:
+            (_resp, body) = self.connection.get_container(
+                container=container,
+                prefix=prefix,
+                limit=limit,
+                marker=marker)
+            return body
+        except ClientException as err:
+            raise SwiftConnectionFailed(reason=err)

smaug/tests/unit/protection/fake_swift_client.py

@@ -0,0 +1,99 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+from swiftclient import ClientException
+import tempfile
+
+
+class FakeSwiftClient(object):
+    def __init__(self, *args, **kwargs):
+        pass
+
+    @classmethod
+    def connection(cls, *args, **kargs):
+        return FakeSwiftConnection()
+
+
+class FakeSwiftConnection(object):
+    def __init__(self, *args, **kwargs):
+        self.swiftdir = tempfile.mkdtemp()
+
+    def put_container(self, container):
+        container_dir = self.swiftdir + "/" + container
+        if os.path.exists(container_dir) is True:
+            return
+        else:
+            os.makedirs(container_dir)
+
+    def get_container(self, container, prefix, limit, marker):
+        container_dir = self.swiftdir + "/" + container
+        body = []
+        if prefix:
+            objects_dir = container_dir + "/" + prefix
+        else:
+            objects_dir = container_dir
+        for f in os.listdir(objects_dir):
+            if os.path.isfile(objects_dir + "/" + f):
+                body.append({"name": f})
+            else:
+                body.append({"subdir": f})
+        return None, body
+
+    def put_object(self, container, obj, contents, headers=None):
+        container_dir = self.swiftdir + "/" + container
+        obj_file = container_dir + "/" + obj
+        obj_dir = obj_file[0:obj_file.rfind("/")]
+        if os.path.exists(container_dir) is True:
+            if os.path.exists(obj_dir) is False:
+                os.makedirs(obj_dir)
+            with open(obj_file, "w") as f:
+                f.write(contents)
+            return
+        else:
+            raise ClientException("error_container")
+
+    def get_object(self, container, obj):
+        container_dir = self.swiftdir + "/" + container
+        obj_file = container_dir + "/" + obj
+        if os.path.exists(container_dir) is True:
+            if os.path.exists(obj_file) is True:
+                with open(obj_file, "r") as f:
+                    return None, f.read()
+            else:
+                raise ClientException("error_obj")
+        else:
+            raise ClientException("error_container")
+
+    def delete_object(self, container, obj):
+        container_dir = self.swiftdir + "/" + container
+        obj_file = container_dir + "/" + obj
+        if os.path.exists(container_dir) is True:
+            if os.path.exists(obj_file) is True:
+                os.remove(obj_file)
+            else:
+                raise ClientException("error_obj")
+        else:
+            raise ClientException("error_container")
+
+    def update_object(self, container, obj, contents):
+        container_dir = self.swiftdir + "/" + container
+        obj_file = container_dir + "/" + obj
+        if os.path.exists(container_dir) is True:
+            if os.path.exists(obj_file) is True:
+                with open(obj_file, "w") as f:
+                    f.write(contents)
+                    return
+            else:
+                raise ClientException("error_obj")
+        else:
+            raise ClientException("error_container")

smaug/tests/unit/protection/test_swift_bank_plugin.py

@@ -0,0 +1,104 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+import os
+from oslo_config import cfg
+from oslo_utils import importutils
+from smaug.tests import base
+from smaug.tests.unit.protection.fake_swift_client import FakeSwiftClient
+from swiftclient import client as swift
+import time
+
+CONF = cfg.CONF
+
+
+class FakeConf(object):
+    def __init__(self):
+        self.lease_expire_window = 600
+        self.lease_renew_window = 120
+        self.lease_validity_window = 100
+
+
+class SwiftBankPluginTest(base.TestCase):
+    def setUp(self):
+        super(SwiftBankPluginTest, self).setUp()
+        self.conf = FakeConf()
+        self.fake_connection = FakeSwiftClient.connection()
+        import_str = "smaug.services.protection.bank_plugins." \
+                     "swift_bank_plugin.SwiftBankPlugin"
+        self.object_container = "objects"
+        swift_bank_plugin_cls = importutils.import_class(
+            import_str=import_str)
+        swift.Connection = mock.MagicMock()
+        swift.Connection.return_value = self.fake_connection
+        self.swift_bank_plugin = swift_bank_plugin_cls(None,
+                                                       self.object_container)
+
+    def test_acquire_lease(self):
+        self.swift_bank_plugin.acquire_lease()
+        expire_time = long(time.time()) + self.conf.lease_expire_window
+        self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
+
+    def test_renew_lease(self):
+        self.swift_bank_plugin.acquire_lease()
+        expire_time = long(time.time()) + self.conf.lease_expire_window
+        self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
+        time.sleep(5)
+        self.swift_bank_plugin.acquire_lease()
+        expire_time = long(time.time()) + self.conf.lease_expire_window
+        self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
+
+    def test_check_lease_validity(self):
+        self.swift_bank_plugin.acquire_lease()
+        expire_time = long(time.time()) + self.conf.lease_expire_window
+        self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
+        is_valid = self.swift_bank_plugin.check_lease_validity()
+        self.assertEqual(is_valid, True)
+
+    def test_create_object(self):
+        self.swift_bank_plugin.create_object("key-1", "value-1")
+        object_file = os.path.join(self.fake_connection.swiftdir,
+                                   self.object_container,
+                                   "key-1")
+        with open(object_file, "r") as f:
+            contents = f.read()
+        self.assertEqual(contents, "value-1")
+
+    def test_delete_object(self):
+        self.swift_bank_plugin.create_object("key", "value")
+        self.swift_bank_plugin.delete_object("key")
+        object_file = os.path.join(self.fake_connection.swiftdir,
+                                   self.object_container,
+                                   "key")
+        self.assertEqual(os.path.isfile(object_file), False)
+
+    def test_get_object(self):
+        self.swift_bank_plugin.create_object("key", "value")
+        value = self.swift_bank_plugin.get_object("key")
+        self.assertEqual(value, "value")
+
+    def test_list_objects(self):
+        self.swift_bank_plugin.create_object("key-1", "value-1")
+        self.swift_bank_plugin.create_object("key-2", "value-2")
+        objects = self.swift_bank_plugin.list_objects(prefix=None)
+        self.assertEqual(len(objects), 2)
+
+    def test_update_object(self):
+        self.swift_bank_plugin.create_object("key-1", "value-1")
+        self.swift_bank_plugin.update_object("key-1", "value-2")
+        object_file = os.path.join(self.fake_connection.swiftdir,
+                                   self.object_container,
+                                   "key-1")
+        with open(object_file, "r") as f:
+            contents = f.read()
+        self.assertEqual(contents, "value-2")

test-requirements.txt

@@ -13,3 +13,4 @@ oslotest>=1.10.0 # Apache-2.0
 testrepository>=0.0.18
 testscenarios>=0.4
 testtools>=1.4.0
+python-swiftclient>=2.2.0 # Apache-2.0