Implement swift bank plugin

A basic implement swift bank plugin

Change-Id: I7c809aed46b225e3de5d298d7c1bb650fab7a139
Closes-Bug: #1545384
This commit is contained in:
smile-luobin 2016-02-14 07:13:03 +00:00 committed by Eran Gampel
parent d4fbc50cfc
commit 4f79808407
9 changed files with 516 additions and 2 deletions

View File

@ -29,3 +29,4 @@ sqlalchemy-migrate>=0.9.6
stevedore>=1.5.0 # Apache-2.0
WebOb>=1.2.3
oslo.i18n>=1.5.0 # Apache-2.0
python-swiftclient>=2.2.0 # Apache-2.0

View File

@ -70,6 +70,16 @@ global_opts = [
choices=['noauth', 'keystone'],
help='The strategy to use for auth. Supports noauth or '
'keystone.'),
cfg.IntOpt('lease_renew_window',
default=120,
help='period for bank lease, in seconds, '
'between bank lease client renew the lease'),
cfg.IntOpt('lease_expire_window',
default=600,
help='expired_window for bank lease, in seconds'),
cfg.IntOpt('lease_validity_window',
default=100,
help='validity_window for bank lease, in seconds'),
]
CONF.register_opts(global_opts)

View File

@ -233,3 +233,31 @@ class ProviderNotFound(NotFound):
class CheckpointNotFound(NotFound):
message = _("Checkpoint %(checkpoint_id)s could"
" not be found.")
class BankCreateObjectFailed(SmaugException):
message = _("Create Object in Bank Failed: %(reason)s")
class BankUpdateObjectFailed(SmaugException):
message = _("Update Object %(key)s in Bank Failed: %(reason)s")
class BankDeleteObjectFailed(SmaugException):
message = _("Delete Object %(key)s in Bank Failed: %(reason)s")
class BankGetObjectFailed(SmaugException):
message = _("Get Object %(key)s in Bank Failed: %(reason)s")
class BankListObjectsFailed(SmaugException):
message = _("Get Object in Bank Failed: %(reason)s")
class AcquireLeaseFailed(SmaugException):
message = _("Acquire Lease in Failed: %(reason)s")
class CreateContainerFailed(SmaugException):
message = _("Create Container in Bank Failed: %(reason)s")

View File

@ -28,12 +28,12 @@ LOG = logging.getLogger(__name__)
@six.add_metaclass(abc.ABCMeta)
class LeasePlugin(object):
@abc.abstractmethod
def acquire_lease(self, owner_id):
def acquire_lease(self):
# TODO(wangliuan)
pass
@abc.abstractmethod
def renew_lease(self, owner_id):
def renew_lease(self):
# TODO(wangliuan)
pass

View File

@ -0,0 +1,271 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_log import log as logging
from oslo_service import loopingcall
from smaug import exception
from smaug.i18n import _, _LE
from smaug.services.protection.bank_plugin import BankPlugin
from smaug.services.protection.bank_plugin import LeasePlugin
from swiftclient import client as swift
from swiftclient import ClientException
import time
import uuid
swift_client_opts = [
cfg.StrOpt('bank_swift_url',
help='The URL of the Swift endpoint'),
cfg.StrOpt('bank_swift_auth_url',
help='The URL of the Keystone endpoint'),
cfg.StrOpt('bank_swift_auth',
default='single_user',
help='Swift authentication mechanism'),
cfg.StrOpt('bank_swift_auth_version',
default='1',
help='Swift authentication version. '
'Specify "1" for auth 1.0, or "2" for auth 2.0'),
cfg.StrOpt('bank_swift_tenant_name',
help='Swift tenant/account name. '
'Required when connecting to an auth 2.0 system'),
cfg.StrOpt('bank_swift_user',
help='Swift user name'),
cfg.StrOpt('bank_swift_key',
help='Swift key for authentication'),
cfg.IntOpt('bank_swift_retry_attempts',
default=3,
help='The number of retries to make for '
'Swift operations'),
cfg.IntOpt('bank_swift_retry_backoff',
default=2,
help='The backoff time in seconds '
'between Swift retries'),
cfg.StrOpt('bank_swift_ca_cert_file',
help='Location of the CA certificate file '
'to use for swift client requests.'),
cfg.BoolOpt('bank_swift_auth_insecure',
default=False,
help='Bypass verification of server certificate when '
'making SSL connection to Swift.'),
]
CONF = cfg.CONF
CONF.register_opts(swift_client_opts, "swift_client")
LOG = logging.getLogger(__name__)
class SwiftConnectionFailed(exception.SmaugException):
message = _("Connection to swift failed: %(reason)s")
class SwiftBankPlugin(BankPlugin, LeasePlugin):
def __init__(self, context, object_container):
super(BankPlugin, self).__init__()
self.context = context
self.swift_retry_attempts = CONF.swift_client.bank_swift_retry_attempts
self.swift_retry_backoff = CONF.swift_client.bank_swift_retry_backoff
self.swift_auth_insecure = CONF.swift_client.bank_swift_auth_insecure
self.swift_ca_cert_file = CONF.swift_client.bank_swift_ca_cert_file
self.lease_expire_window = CONF.lease_expire_window
self.lease_renew_window = CONF.lease_renew_window
# TODO(luobin):
# init lease_validity_window
# according to lease_renew_window if not configured
self.lease_validity_window = CONF.lease_validity_window
# TODO(luobin): create a uuid of this bank_plugin
self.owner_id = str(uuid.uuid4())
self.lease_expire_time = 0
self.bank_leases_container = "leases"
self.bank_object_container = object_container
self.connection = self._setup_connection()
# create container
try:
self._put_container(self.bank_object_container)
self._put_container(self.bank_leases_container)
except SwiftConnectionFailed as err:
LOG.error(_LE("bank plugin create container failed."))
raise exception.CreateContainerFailed(reason=err)
# acquire lease
try:
self.acquire_lease()
except exception.AcquireLeaseFailed as err:
LOG.error(_LE("bank plugin acquire lease failed."))
raise err
# start renew lease
renew_lease_loop = loopingcall.FixedIntervalLoopingCall(
self.renew_lease)
renew_lease_loop.start(interval=self.lease_renew_window,
initial_delay=self.lease_renew_window)
def _setup_connection(self):
if CONF.swift_client.bank_swift_auth == "single_user":
connection = swift.Connection(
authurl=CONF.swift_client.bank_swift_auth_url,
auth_version=CONF.swift_client.bank_swift_auth_version,
tenant_name=CONF.swift_client.bank_swift_tenant_name,
user=CONF.swift_client.bank_swift_user,
key=CONF.swift_client.bank_swift_key,
retries=self.swift_retry_attempts,
starting_backoff=self.swift_retry_backoff,
insecure=self.swift_auth_insecure,
cacert=self.swift_ca_cert_file)
else:
connection = swift.Connection(
preauthurl=CONF.swift_client.bank_swift_url,
preauthtoken=self.context.auth_token,
retries=self.swift_retry_attempts,
starting_backoff=self.swift_retry_backoff,
insecure=self.swift_auth_insecure,
cacert=self.swift_ca_cert_file)
return connection
def create_object(self, key, value):
try:
self._put_object(container=self.bank_object_container,
obj=key,
contents=value)
except SwiftConnectionFailed as err:
LOG.error(_LE("create object failed, err: %s."), err)
raise exception.BankCreateObjectFailed(reasone=err,
key=key)
def update_object(self, key, value):
try:
self._put_object(container=self.bank_object_container,
obj=key,
contents=value)
except SwiftConnectionFailed as err:
LOG.error(_LE("update object failed, err: %s."), err)
raise exception.BankUpdateObjectFailed(reasone=err,
key=key)
def delete_object(self, key):
try:
self._delete_object(container=self.bank_object_container,
obj=key)
except SwiftConnectionFailed as err:
LOG.error(_LE("delete object failed, err: %s."), err)
raise exception.BankDeleteObjectFailed(reasone=err,
key=key)
def get_object(self, key):
try:
return self._get_object(container=self.bank_object_container,
obj=key)
except SwiftConnectionFailed as err:
LOG.error(_LE("get object failed, err: %s."), err)
raise exception.BankGetObjectFailed(reasone=err,
key=key)
def list_objects(self, prefix=None, limit=None, marker=None):
object_names = []
try:
body = self._get_container(container=self.bank_object_container,
prefix=prefix,
limit=limit,
marker=marker)
except SwiftConnectionFailed as err:
LOG.error(_LE("list objects failed, err: %s."), err)
raise exception.BankListObjectsFailed(reasone=err)
for obj in body:
if obj.get("name"):
object_names.append(obj.get("name"))
return object_names
def acquire_lease(self):
container = self.bank_leases_container
obj = self.owner_id
contents = self.owner_id
headers = {'X-Delete-After': self.lease_expire_window}
try:
self._put_object(container=container,
obj=obj,
contents=contents,
headers=headers)
self.lease_expire_time = long(
time.time()) + self.lease_expire_window
except SwiftConnectionFailed as err:
LOG.error(_LE("acquire lease failed, err:%s."), err)
raise exception.AcquireLeaseFailed(reason=err)
def renew_lease(self):
container = self.bank_leases_container
obj = self.owner_id
headers = {'X-Delete-After': self.lease_expire_window}
try:
self._post_object(container=container,
obj=obj,
headers=headers)
self.lease_expire_time = long(
time.time()) + self.lease_expire_window
except SwiftConnectionFailed as err:
LOG.error(_LE("acquire lease failed, err:%s."), err)
def check_lease_validity(self):
if (self.lease_expire_time - long(time.time()) >=
self.lease_validity_window):
return True
else:
return False
def _put_object(self, container, obj, contents, headers=None):
try:
self.connection.put_object(container=container,
obj=obj,
contents=contents,
headers=headers)
except ClientException as err:
raise SwiftConnectionFailed(reason=err)
def _get_object(self, container, obj):
try:
(_resp, body) = self.connection.get_object(container=container,
obj=obj)
return body
except ClientException as err:
raise SwiftConnectionFailed(reason=err)
def _post_object(self, container, obj, headers):
try:
self.connection.post_object(container=container,
obj=obj,
headers=headers)
except ClientException as err:
raise SwiftConnectionFailed(reason=err)
def _delete_object(self, container, obj):
try:
self.connection.delete_object(container=container,
obj=obj)
except ClientException as err:
raise SwiftConnectionFailed(reason=err)
def _put_container(self, container):
try:
self.connection.put_container(container=container)
except ClientException as err:
raise SwiftConnectionFailed(reason=err)
def _get_container(self, container, prefix=None, limit=None, marker=None):
try:
(_resp, body) = self.connection.get_container(
container=container,
prefix=prefix,
limit=limit,
marker=marker)
return body
except ClientException as err:
raise SwiftConnectionFailed(reason=err)

View File

@ -0,0 +1,99 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from swiftclient import ClientException
import tempfile
class FakeSwiftClient(object):
def __init__(self, *args, **kwargs):
pass
@classmethod
def connection(cls, *args, **kargs):
return FakeSwiftConnection()
class FakeSwiftConnection(object):
def __init__(self, *args, **kwargs):
self.swiftdir = tempfile.mkdtemp()
def put_container(self, container):
container_dir = self.swiftdir + "/" + container
if os.path.exists(container_dir) is True:
return
else:
os.makedirs(container_dir)
def get_container(self, container, prefix, limit, marker):
container_dir = self.swiftdir + "/" + container
body = []
if prefix:
objects_dir = container_dir + "/" + prefix
else:
objects_dir = container_dir
for f in os.listdir(objects_dir):
if os.path.isfile(objects_dir + "/" + f):
body.append({"name": f})
else:
body.append({"subdir": f})
return None, body
def put_object(self, container, obj, contents, headers=None):
container_dir = self.swiftdir + "/" + container
obj_file = container_dir + "/" + obj
obj_dir = obj_file[0:obj_file.rfind("/")]
if os.path.exists(container_dir) is True:
if os.path.exists(obj_dir) is False:
os.makedirs(obj_dir)
with open(obj_file, "w") as f:
f.write(contents)
return
else:
raise ClientException("error_container")
def get_object(self, container, obj):
container_dir = self.swiftdir + "/" + container
obj_file = container_dir + "/" + obj
if os.path.exists(container_dir) is True:
if os.path.exists(obj_file) is True:
with open(obj_file, "r") as f:
return None, f.read()
else:
raise ClientException("error_obj")
else:
raise ClientException("error_container")
def delete_object(self, container, obj):
container_dir = self.swiftdir + "/" + container
obj_file = container_dir + "/" + obj
if os.path.exists(container_dir) is True:
if os.path.exists(obj_file) is True:
os.remove(obj_file)
else:
raise ClientException("error_obj")
else:
raise ClientException("error_container")
def update_object(self, container, obj, contents):
container_dir = self.swiftdir + "/" + container
obj_file = container_dir + "/" + obj
if os.path.exists(container_dir) is True:
if os.path.exists(obj_file) is True:
with open(obj_file, "w") as f:
f.write(contents)
return
else:
raise ClientException("error_obj")
else:
raise ClientException("error_container")

View File

@ -0,0 +1,104 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
import os
from oslo_config import cfg
from oslo_utils import importutils
from smaug.tests import base
from smaug.tests.unit.protection.fake_swift_client import FakeSwiftClient
from swiftclient import client as swift
import time
CONF = cfg.CONF
class FakeConf(object):
def __init__(self):
self.lease_expire_window = 600
self.lease_renew_window = 120
self.lease_validity_window = 100
class SwiftBankPluginTest(base.TestCase):
def setUp(self):
super(SwiftBankPluginTest, self).setUp()
self.conf = FakeConf()
self.fake_connection = FakeSwiftClient.connection()
import_str = "smaug.services.protection.bank_plugins." \
"swift_bank_plugin.SwiftBankPlugin"
self.object_container = "objects"
swift_bank_plugin_cls = importutils.import_class(
import_str=import_str)
swift.Connection = mock.MagicMock()
swift.Connection.return_value = self.fake_connection
self.swift_bank_plugin = swift_bank_plugin_cls(None,
self.object_container)
def test_acquire_lease(self):
self.swift_bank_plugin.acquire_lease()
expire_time = long(time.time()) + self.conf.lease_expire_window
self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
def test_renew_lease(self):
self.swift_bank_plugin.acquire_lease()
expire_time = long(time.time()) + self.conf.lease_expire_window
self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
time.sleep(5)
self.swift_bank_plugin.acquire_lease()
expire_time = long(time.time()) + self.conf.lease_expire_window
self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
def test_check_lease_validity(self):
self.swift_bank_plugin.acquire_lease()
expire_time = long(time.time()) + self.conf.lease_expire_window
self.assertEqual(self.swift_bank_plugin.lease_expire_time, expire_time)
is_valid = self.swift_bank_plugin.check_lease_validity()
self.assertEqual(is_valid, True)
def test_create_object(self):
self.swift_bank_plugin.create_object("key-1", "value-1")
object_file = os.path.join(self.fake_connection.swiftdir,
self.object_container,
"key-1")
with open(object_file, "r") as f:
contents = f.read()
self.assertEqual(contents, "value-1")
def test_delete_object(self):
self.swift_bank_plugin.create_object("key", "value")
self.swift_bank_plugin.delete_object("key")
object_file = os.path.join(self.fake_connection.swiftdir,
self.object_container,
"key")
self.assertEqual(os.path.isfile(object_file), False)
def test_get_object(self):
self.swift_bank_plugin.create_object("key", "value")
value = self.swift_bank_plugin.get_object("key")
self.assertEqual(value, "value")
def test_list_objects(self):
self.swift_bank_plugin.create_object("key-1", "value-1")
self.swift_bank_plugin.create_object("key-2", "value-2")
objects = self.swift_bank_plugin.list_objects(prefix=None)
self.assertEqual(len(objects), 2)
def test_update_object(self):
self.swift_bank_plugin.create_object("key-1", "value-1")
self.swift_bank_plugin.update_object("key-1", "value-2")
object_file = os.path.join(self.fake_connection.swiftdir,
self.object_container,
"key-1")
with open(object_file, "r") as f:
contents = f.read()
self.assertEqual(contents, "value-2")

View File

@ -13,3 +13,4 @@ oslotest>=1.10.0 # Apache-2.0
testrepository>=0.0.18
testscenarios>=0.4
testtools>=1.4.0
python-swiftclient>=2.2.0 # Apache-2.0