Merge "The file system based bank implementation"
This commit is contained in:
commit
5f4fe69897
|
@ -0,0 +1,190 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import errno
|
||||
import os
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
from oslo_serialization import jsonutils
|
||||
from oslo_utils import uuidutils
|
||||
|
||||
from karbor import exception
|
||||
from karbor.i18n import _
|
||||
from karbor.services.protection.bank_plugin import BankPlugin
|
||||
|
||||
import six
|
||||
|
||||
file_system_bank_plugin_opts = [
|
||||
cfg.StrOpt('file_system_bank_path',
|
||||
help='The file system bank path to use.'),
|
||||
cfg.StrOpt('bank_object_container',
|
||||
default='karbor',
|
||||
help='The file system bank container to use.'),
|
||||
]
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileSystemBankPlugin(BankPlugin):
|
||||
"""File system bank plugin"""
|
||||
def __init__(self, config):
|
||||
super(FileSystemBankPlugin, self).__init__(config)
|
||||
self._config.register_opts(file_system_bank_plugin_opts,
|
||||
"file_system_bank_plugin")
|
||||
plugin_cfg = self._config.file_system_bank_plugin
|
||||
self.file_system_bank_path = plugin_cfg.file_system_bank_path
|
||||
self.bank_object_container = plugin_cfg.bank_object_container
|
||||
|
||||
try:
|
||||
self._create_dir(self.file_system_bank_path)
|
||||
self.object_container_path = "/".join([self.file_system_bank_path,
|
||||
self.bank_object_container])
|
||||
self._create_dir(self.object_container_path)
|
||||
except OSError as err:
|
||||
LOG.exception(_("Init file system bank failed. err: %s"), err)
|
||||
|
||||
self.owner_id = uuidutils.generate_uuid()
|
||||
|
||||
def _validate_path(self, path):
|
||||
if path.find('..') >= 0:
|
||||
msg = (_("The path(%s) is invalid.") % path)
|
||||
raise exception.InvalidInput(msg)
|
||||
|
||||
def _create_dir(self, path):
|
||||
try:
|
||||
original_umask = None
|
||||
try:
|
||||
original_umask = os.umask(0)
|
||||
os.makedirs(path)
|
||||
finally:
|
||||
os.umask(original_umask)
|
||||
except OSError as err:
|
||||
if err.errno == errno.EEXIST and os.path.isdir(path):
|
||||
pass
|
||||
else:
|
||||
LOG.exception(_("Create the directory failed. path: %s"), path)
|
||||
raise
|
||||
|
||||
def _write_object(self, path, data):
|
||||
obj_file_name = None
|
||||
try:
|
||||
obj_path = self.object_container_path + path.rsplit('/', 1)[0]
|
||||
obj_file_name = self.object_container_path + path
|
||||
self._create_dir(obj_path)
|
||||
mode = "wb"
|
||||
if isinstance(data, six.string_types):
|
||||
mode = "w"
|
||||
with open(obj_file_name, mode=mode) as obj_file:
|
||||
obj_file.write(data)
|
||||
except (OSError, IOError):
|
||||
LOG.exception(_("Write object failed. name: %s"), obj_file_name)
|
||||
raise
|
||||
|
||||
def _get_object(self, path):
|
||||
obj_file_name = self.object_container_path + path
|
||||
if not os.path.isfile(obj_file_name):
|
||||
LOG.exception(_("Object is not a file. name: %s"), obj_file_name)
|
||||
raise
|
||||
try:
|
||||
with open(obj_file_name, mode='r') as obj_file:
|
||||
data = obj_file.read()
|
||||
return data
|
||||
except OSError:
|
||||
LOG.exception(_("Get object failed. name: %s"), obj_file_name)
|
||||
raise
|
||||
|
||||
def _delete_object(self, path):
|
||||
obj_path = self.object_container_path + path.rsplit('/', 1)[0]
|
||||
obj_file_name = self.object_container_path + path
|
||||
try:
|
||||
os.remove(obj_file_name)
|
||||
if not os.listdir(obj_path) and (
|
||||
obj_path != self.object_container_path):
|
||||
os.rmdir(obj_path)
|
||||
except OSError:
|
||||
LOG.exception(_("Delete the object failed. name: %s"),
|
||||
obj_file_name)
|
||||
raise
|
||||
|
||||
def _list_object(self, path):
|
||||
obj_file_path = self.object_container_path + path
|
||||
if not os.path.isdir(obj_file_path):
|
||||
LOG.exception(_("Path is not a directory. name: %s"),
|
||||
obj_file_path)
|
||||
raise
|
||||
try:
|
||||
if os.path.isdir(obj_file_path):
|
||||
return os.listdir(obj_file_path)
|
||||
else:
|
||||
base_dir = os.path.dirname(obj_file_path)
|
||||
base_name = os.path.basename(obj_file_path)
|
||||
return (
|
||||
base_dir + '/' + obj_file
|
||||
for obj_file in os.listdir(base_dir)
|
||||
if obj_file.startswith(base_name)
|
||||
)
|
||||
except OSError:
|
||||
LOG.exception(_("List the object failed. path: %s"), obj_file_path)
|
||||
raise
|
||||
|
||||
def get_owner_id(self):
|
||||
return self.owner_id
|
||||
|
||||
def update_object(self, key, value):
|
||||
LOG.debug("FsBank: update_object. key: %s", key)
|
||||
self._validate_path(key)
|
||||
try:
|
||||
if not isinstance(value, str):
|
||||
value = jsonutils.dumps(value)
|
||||
self._write_object(path=key,
|
||||
data=value)
|
||||
except OSError as err:
|
||||
LOG.error("Update object failed. err: %s", err)
|
||||
raise exception.BankUpdateObjectFailed(reason=err,
|
||||
key=key)
|
||||
|
||||
def delete_object(self, key):
|
||||
LOG.debug("FsBank: delete_object. key: %s", key)
|
||||
self._validate_path(key)
|
||||
try:
|
||||
self._delete_object(path=key)
|
||||
except OSError as err:
|
||||
LOG.error("Delete object failed. err: %s", err)
|
||||
raise exception.BankDeleteObjectFailed(reason=err,
|
||||
key=key)
|
||||
|
||||
def get_object(self, key):
|
||||
LOG.debug("FsBank: get_object. key: %s", key)
|
||||
self._validate_path(key)
|
||||
try:
|
||||
data = self._get_object(path=key)
|
||||
except OSError as err:
|
||||
LOG.error("Get object failed. err: %s", err)
|
||||
raise exception.BankGetObjectFailed(reason=err,
|
||||
key=key)
|
||||
if isinstance(data, six.string_types):
|
||||
try:
|
||||
data = jsonutils.loads(data)
|
||||
except ValueError:
|
||||
pass
|
||||
return data
|
||||
|
||||
def list_objects(self, prefix=None, limit=None, marker=None,
|
||||
sort_dir=None):
|
||||
LOG.debug("FsBank: list_objects. key: %s", prefix)
|
||||
try:
|
||||
file_lists = self._list_object(prefix)
|
||||
return (
|
||||
file_lists[-limit:] if limit is not None else file_lists)
|
||||
except OSError as err:
|
||||
LOG.error("List objects failed. err: %s", err)
|
||||
raise exception.BankListObjectsFailed(reason=err)
|
|
@ -0,0 +1,82 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from oslo_config import cfg
|
||||
from oslo_config import fixture
|
||||
from oslo_utils import importutils
|
||||
|
||||
from karbor import exception
|
||||
from karbor.tests import base
|
||||
|
||||
|
||||
class FileSystemBankPluginTest(base.TestCase):
|
||||
def setUp(self):
|
||||
super(FileSystemBankPluginTest, self).setUp()
|
||||
|
||||
import_str = (
|
||||
"karbor.services.protection.bank_plugins."
|
||||
"file_system_bank_plugin.FileSystemBankPlugin"
|
||||
)
|
||||
plugin_config = cfg.ConfigOpts()
|
||||
plugin_config_fixture = self.useFixture(fixture.Config(plugin_config))
|
||||
plugin_config_fixture.load_raw_values(
|
||||
group='file_system_bank_plugin',
|
||||
file_system_bank_path=tempfile.mkdtemp(),
|
||||
)
|
||||
fs_bank_plugin_cls = importutils.import_class(
|
||||
import_str=import_str)
|
||||
self.fs_bank_plugin = fs_bank_plugin_cls(plugin_config)
|
||||
|
||||
def test_delete_object(self):
|
||||
self.fs_bank_plugin.update_object("/key", "value")
|
||||
self.fs_bank_plugin.delete_object("/key")
|
||||
object_file = (
|
||||
self.fs_bank_plugin.object_container_path + "/key")
|
||||
self.assertEqual(os.path.isfile(object_file), False)
|
||||
|
||||
def test_get_object(self):
|
||||
self.fs_bank_plugin.update_object("/key", "value")
|
||||
value = self.fs_bank_plugin.get_object("/key")
|
||||
self.assertEqual(value, "value")
|
||||
|
||||
def test_list_objects(self):
|
||||
self.fs_bank_plugin.update_object("/list/key-1", "value-1")
|
||||
self.fs_bank_plugin.update_object("/list/key-2", "value-2")
|
||||
objects = self.fs_bank_plugin.list_objects(prefix="/list")
|
||||
self.assertEqual(len(objects), 2)
|
||||
self.assertIn('key-1', objects)
|
||||
self.assertIn('key-2', objects)
|
||||
|
||||
def test_update_object(self):
|
||||
self.fs_bank_plugin.update_object("/key-1", "value-1")
|
||||
self.fs_bank_plugin.update_object("/key-1", "value-2")
|
||||
object_file = (
|
||||
self.fs_bank_plugin.object_container_path + "/key-1")
|
||||
with open(object_file, "r") as f:
|
||||
contents = f.read()
|
||||
self.assertEqual(contents, "value-2")
|
||||
|
||||
def test_update_object_with_invaild_path(self):
|
||||
self.assertRaises(exception.InvalidInput,
|
||||
self.fs_bank_plugin.update_object,
|
||||
"../../../../../../../etc/shadow",
|
||||
"value-1")
|
||||
|
||||
def test_create_get_dict_object(self):
|
||||
self.fs_bank_plugin.update_object("/index.json",
|
||||
{"key": "value"})
|
||||
value = self.fs_bank_plugin.get_object(
|
||||
"/index.json")
|
||||
self.assertEqual(value, {"key": "value"})
|
Loading…
Reference in New Issue