Provide private data storage API for drivers

- Add implementation of private data storage API
- Add appropriate unit tests

Partially implements bp private-data-storage-api-for-drivers

Change-Id: I61b3cc41f545b540101819feb603e45d0a057a4e
This commit is contained in:
Igor Malinovskiy 2015-05-26 17:53:00 +03:00
parent 94e8c921db
commit 8953627967
10 changed files with 666 additions and 1 deletions

View File

@ -756,3 +756,20 @@ def share_type_extra_specs_update_or_create(context, share_type_id,
return IMPL.share_type_extra_specs_update_or_create(context,
share_type_id,
extra_specs)
def driver_private_data_get(context, host, entity_id, key=None, default=None):
"""Get one, list or all key-value pairs for given host and entity_id."""
return IMPL.driver_private_data_get(context, host, entity_id, key, default)
def driver_private_data_update(context, host, entity_id, details,
delete_existing=False):
"""Update key-value pairs for given host and entity_id."""
return IMPL.driver_private_data_update(context, host, entity_id, details,
delete_existing)
def driver_private_data_delete(context, host, entity_id, key=None):
"""Remove one, list or all key-value pairs for given host and entity_id."""
return IMPL.driver_private_data_delete(context, host, entity_id, key)

View File

@ -0,0 +1,67 @@
# Copyright 2015 Mirantis inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add_driver_private_data_table
Revision ID: 3a482171410f
Revises: 56cdbe267881
Create Date: 2015-04-21 14:47:38.201658
"""
# revision identifiers, used by Alembic.
revision = '3a482171410f'
down_revision = '56cdbe267881'
from alembic import op
from oslo_log import log
import sqlalchemy as sql
from manila.i18n import _LE
LOG = log.getLogger(__name__)
drivers_private_data_table_name = 'drivers_private_data'
def upgrade():
try:
op.create_table(
drivers_private_data_table_name,
sql.Column('created_at', sql.DateTime),
sql.Column('updated_at', sql.DateTime),
sql.Column('deleted_at', sql.DateTime),
sql.Column('deleted', sql.Integer, default=0),
sql.Column('host', sql.String(255),
nullable=False, primary_key=True),
sql.Column('entity_uuid', sql.String(36),
nullable=False, primary_key=True),
sql.Column('key', sql.String(255),
nullable=False, primary_key=True),
sql.Column('value', sql.String(1023), nullable=False),
mysql_engine='InnoDB',
)
except Exception:
LOG.error(_LE("Table |%s| not created!"),
drivers_private_data_table_name)
raise
def downgrade():
try:
op.drop_table(drivers_private_data_table_name)
except Exception:
LOG.error(_LE("%s table not dropped"), drivers_private_data_table_name)
raise

View File

@ -18,6 +18,7 @@
"""Implementation of SQLAlchemy backend."""
import copy
import datetime
import sys
import uuid
@ -2086,6 +2087,97 @@ def share_server_backend_details_get(context, share_server_id,
return dict([(item.key, item.value) for item in query])
###################
def _driver_private_data_query(session, context, host, entity_id, key=None,
read_deleted=False):
query = model_query(context, models.DriverPrivateData,
session=session, read_deleted=read_deleted)\
.filter_by(host=host)\
.filter_by(entity_uuid=entity_id)
if isinstance(key, list):
return query.filter(models.DriverPrivateData.key.in_(key))
elif key is not None:
return query.filter_by(key=key)
return query
@require_context
def driver_private_data_get(context, host, entity_id, key=None,
default=None, session=None):
if not session:
session = get_session()
query = _driver_private_data_query(session, context, host, entity_id, key)
if key is None or isinstance(key, list):
return dict([(item.key, item.value) for item in query.all()])
else:
result = query.first()
return result["value"] if result is not None else default
@require_context
def driver_private_data_update(context, host, entity_id, details,
delete_existing=False, session=None):
# NOTE(u_glide): following code modifies details dict, that's why we should
# copy it
new_details = copy.deepcopy(details)
if not session:
session = get_session()
with session.begin():
# Process existing data
# NOTE(u_glide): read_deleted=None means here 'read all'
original_data = _driver_private_data_query(
session, context, host, entity_id, read_deleted=None).all()
for data_ref in original_data:
in_new_details = data_ref['key'] in new_details
if in_new_details:
new_value = six.text_type(new_details.pop(data_ref['key']))
data_ref.update({
"value": new_value,
"deleted": 0,
"deleted_at": None
})
data_ref.save(session=session)
elif delete_existing and data_ref['deleted'] != 1:
data_ref.update({
"deleted": 1, "deleted_at": timeutils.utcnow()
})
data_ref.save(session=session)
# Add new data
for key, value in new_details.items():
data_ref = models.DriverPrivateData()
data_ref.update({
"host": host,
"entity_uuid": entity_id,
"key": key,
"value": six.text_type(value)
})
data_ref.save(session=session)
return details
@require_context
def driver_private_data_delete(context, host, entity_id, key=None,
session=None):
if not session:
session = get_session()
with session.begin():
query = _driver_private_data_query(session, context, host,
entity_id, key)
query.update({"deleted": 1, "deleted_at": timeutils.utcnow()})
###################

View File

@ -457,6 +457,15 @@ class NetworkAllocation(BASE, ManilaBase):
default=constants.STATUS_NEW)
class DriverPrivateData(BASE, ManilaBase):
"""Represents a private data as key-value pairs for a driver."""
__tablename__ = 'drivers_private_data'
host = Column(String(255), nullable=False, primary_key=True)
entity_uuid = Column(String(36), nullable=False, primary_key=True)
key = Column(String(255), nullable=False, primary_key=True)
value = Column(String(1023), nullable=False)
def register_models():
"""Register Models and create metadata.

View File

@ -65,6 +65,7 @@ import manila.share.drivers.netapp.options
import manila.share.drivers.quobyte.quobyte
import manila.share.drivers.service_instance
import manila.share.drivers.zfssa.zfssashare
import manila.share.drivers_private_data
import manila.share.manager
import manila.volume
import manila.volume.cinder
@ -114,6 +115,7 @@ _global_opt_lists = [
manila.share.driver.ganesha_opts,
manila.share.driver.share_opts,
manila.share.driver.ssh_opts,
manila.share.drivers_private_data.private_data_opts,
manila.share.drivers.emc.driver.EMC_NAS_OPTS,
manila.share.drivers.emc.plugins.isilon.isilon.ISILON_OPTS,
manila.share.drivers.generic.share_opts,

View File

@ -0,0 +1,176 @@
# Copyright 2015 Mirantis inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Module provides possibility for share drivers to store private information
related to common Manila models like Share or Snapshot.
"""
import abc
from oslo_config import cfg
from oslo_utils import importutils
from oslo_utils import uuidutils
import six
from manila.db import api as db_api
from manila.i18n import _
private_data_opts = [
cfg.StrOpt(
'drivers_private_storage_class',
default='manila.share.drivers_private_data.SqlStorageDriver',
help='The full class name of the Private Data Driver class to use.'),
]
CONF = cfg.CONF
@six.add_metaclass(abc.ABCMeta)
class StorageDriver(object):
def __init__(self, context, backend_host):
# Backend shouldn't access data stored by another backend
self.backend_host = backend_host
self.context = context
@abc.abstractmethod
def get(self, entity_id, key, default):
"""Backend implementation for DriverPrivateData.get() method.
Should return all keys for given 'entity_id' if 'key' is None.
Otherwise should return value for provided 'key'.
If values for provided 'entity_id' or 'key' not found,
should return 'default'.
See DriverPrivateData.get() method for more details.
"""
@abc.abstractmethod
def update(self, entity_id, details, delete_existing):
"""Backend implementation for DriverPrivateData.update() method.
Should update details for given 'entity_id' with behaviour defined
by 'delete_existing' boolean flag.
See DriverPrivateData.update() method for more details.
"""
@abc.abstractmethod
def delete(self, entity_id, key):
"""Backend implementation for DriverPrivateData.delete() method.
Should return delete all keys if 'key' is None.
Otherwise should delete value for provided 'key'.
See DriverPrivateData.update() method for more details.
"""
class SqlStorageDriver(StorageDriver):
def update(self, entity_id, details, delete_existing):
return db_api.driver_private_data_update(
self.context, self.backend_host, entity_id, details,
delete_existing
)
def get(self, entity_id, key, default):
return db_api.driver_private_data_get(
self.context, self.backend_host, entity_id, key, default
)
def delete(self, entity_id, key):
return db_api.driver_private_data_delete(
self.context, self.backend_host, entity_id, key
)
class DriverPrivateData(object):
def __init__(self, storage=None, *args, **kwargs):
"""Init method.
:param storage: None or inheritor of StorageDriver abstract class
:param config_group: Optional -- Config group used for loading settings
:param context: Optional -- Current context
:param backend_host: Optional -- Driver host
"""
config_group_name = kwargs.get('config_group')
CONF.register_opts(private_data_opts, group=config_group_name)
if storage is not None:
self._storage = storage
elif 'context' in kwargs and 'backend_host' in kwargs:
if config_group_name:
conf = getattr(CONF, config_group_name)
else:
conf = CONF
storage_class = conf.drivers_private_storage_class
cls = importutils.import_class(storage_class)
self._storage = cls(kwargs.get('context'),
kwargs.get('backend_host'))
else:
msg = _("You should provide 'storage' parameter or"
" 'context' and 'backend_host' parameters.")
raise ValueError(msg)
def get(self, entity_id, key=None, default=None):
"""Get one, list or all key-value pairs.
:param entity_id: Model UUID
:param key: Key string or list of keys
:param default: Default value for case when key(s) not found
:returns: string or dict
"""
self._validate_entity_id(entity_id)
return self._storage.get(entity_id, key, default)
def update(self, entity_id, details, delete_existing=False):
"""Update or create specified key-value pairs.
:param entity_id: Model UUID
:param details: dict with key-value pairs data. Keys and values should
be strings.
:param delete_existing: boolean flag which determines behaviour
for existing key-value pairs:
True - remove all existing key-value pairs
False (default) - leave as is
"""
self._validate_entity_id(entity_id)
if not isinstance(details, dict):
msg = (_("Provided details %s is not valid dict.")
% six.text_type(details))
raise ValueError(msg)
return self._storage.update(
entity_id, details, delete_existing)
def delete(self, entity_id, key=None):
"""Delete one, list or all key-value pairs.
:param entity_id: Model UUID
:param key: Key string or list of keys
"""
self._validate_entity_id(entity_id)
return self._storage.delete(entity_id, key)
@staticmethod
def _validate_entity_id(entity_id):
if not uuidutils.is_uuid_like(entity_id):
msg = (_("Provided entity_id %s is not valid UUID.")
% six.text_type(entity_id))
raise ValueError(msg)

View File

@ -39,6 +39,7 @@ from manila.i18n import _LW
from manila import manager
from manila import quota
import manila.share.configuration
from manila.share import drivers_private_data
from manila.share import utils as share_utils
from manila import utils
@ -112,8 +113,16 @@ class ShareManager(manager.SchedulerDependentManager):
msg_args)
share_driver = MAPPING[share_driver]
ctxt = context.get_admin_context()
private_storage = drivers_private_data.DriverPrivateData(
context=ctxt, backend_host=self.host,
config_group=self.configuration.config_group
)
self.driver = importutils.import_object(
share_driver, configuration=self.configuration)
share_driver, private_storage=private_storage,
configuration=self.configuration
)
def _ensure_share_has_pool(self, ctxt, share):
pool = share_utils.extract_host(share['host'], 'pool')

View File

@ -15,11 +15,16 @@
"""Testing of SQLAlchemy backend."""
import ddt
from oslo_utils import uuidutils
import six
from manila import context
from manila.db.sqlalchemy import api
from manila import test
@ddt.ddt
class SQLAlchemyAPIShareTestCase(test.TestCase):
def setUp(self):
@ -76,3 +81,94 @@ class SQLAlchemyAPIShareTestCase(test.TestCase):
actual_result = api.share_export_locations_get(self.ctxt, share['id'])
self.assertTrue(actual_result == [initial_location])
def _get_driver_test_data(self):
return ("fake@host", uuidutils.generate_uuid())
@ddt.data({"details": {"foo": "bar", "tee": "too"},
"valid": {"foo": "bar", "tee": "too"}},
{"details": {"foo": "bar", "tee": ["test"]},
"valid": {"foo": "bar", "tee": six.text_type(["test"])}})
@ddt.unpack
def test_driver_private_data_update(self, details, valid):
test_host, test_id = self._get_driver_test_data()
initial_data = api.driver_private_data_get(
self.ctxt, test_host, test_id)
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
actual_data = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual({}, initial_data)
self.assertEqual(valid, actual_data)
def test_driver_private_data_update_with_duplicate(self):
test_host, test_id = self._get_driver_test_data()
details = {"tee": "too"}
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
actual_result = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual(details, actual_result)
def test_driver_private_data_update_with_delete_existing(self):
test_host, test_id = self._get_driver_test_data()
details = {"key1": "val1", "key2": "val2", "key3": "val3"}
details_update = {"key1": "val1_upd", "key4": "new_val"}
# Create new details
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
api.driver_private_data_update(self.ctxt, test_host, test_id,
details_update, delete_existing=True)
actual_result = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual(details_update, actual_result)
def test_driver_private_data_get(self):
test_host, test_id = self._get_driver_test_data()
test_key = "foo"
test_keys = [test_key, "tee"]
details = {test_keys[0]: "val", test_keys[1]: "val", "mee": "foo"}
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
actual_result_all = api.driver_private_data_get(
self.ctxt, test_host, test_id)
actual_result_single_key = api.driver_private_data_get(
self.ctxt, test_host, test_id, test_key)
actual_result_list = api.driver_private_data_get(
self.ctxt, test_host, test_id, test_keys)
self.assertEqual(details, actual_result_all)
self.assertEqual(details[test_key], actual_result_single_key)
self.assertEqual(dict.fromkeys(test_keys, "val"), actual_result_list)
def test_driver_private_data_delete_single(self):
test_host, test_id = self._get_driver_test_data()
test_key = "foo"
details = {test_key: "bar", "tee": "too"}
valid_result = {"tee": "too"}
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
api.driver_private_data_delete(self.ctxt, test_host, test_id, test_key)
actual_result = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual(valid_result, actual_result)
def test_driver_private_data_delete_all(self):
test_host, test_id = self._get_driver_test_data()
details = {"foo": "bar", "tee": "too"}
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
api.driver_private_data_delete(self.ctxt, test_host, test_id)
actual_result = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual({}, actual_result)

View File

@ -0,0 +1,179 @@
# Copyright 2015 Mirantis inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ddt
import mock
from oslo_utils import uuidutils
from manila.share import drivers_private_data as pd
from manila import test
@ddt.ddt
class DriverPrivateDataTestCase(test.TestCase):
"""Tests DriverPrivateData."""
def setUp(self):
super(DriverPrivateDataTestCase, self).setUp()
self.fake_storage = mock.Mock()
self.entity_id = uuidutils.generate_uuid()
def test_default_storage_driver(self):
private_data = pd.DriverPrivateData(
storage=None, context="fake", backend_host="fake")
self.assertIsInstance(private_data._storage, pd.SqlStorageDriver)
def test_custom_storage_driver(self):
private_data = pd.DriverPrivateData(storage=self.fake_storage)
self.assertEqual(private_data._storage, self.fake_storage)
def test_invalid_parameters(self):
self.assertRaises(ValueError, pd.DriverPrivateData)
@ddt.data({'context': 'fake'}, {'backend_host': 'fake'})
def test_invalid_single_parameter(self, test_args):
self.assertRaises(ValueError, pd.DriverPrivateData, **test_args)
@ddt.data("111", ["fake"], None)
def test_validate_entity_id_invalid(self, entity_id):
data = pd.DriverPrivateData(storage="fake")
self.assertRaises(ValueError, data._validate_entity_id, entity_id)
def test_validate_entity_id_valid(self):
actual_result = (
pd.DriverPrivateData._validate_entity_id(self.entity_id)
)
self.assertIsNone(actual_result)
def test_update(self):
data = pd.DriverPrivateData(storage=self.fake_storage)
details = {"foo": "bar"}
self.mock_object(self.fake_storage, 'update',
mock.Mock(return_value=True))
actual_result = data.update(
self.entity_id,
details,
delete_existing=True
)
self.assertTrue(actual_result)
self.fake_storage.update.assert_called_once_with(
self.entity_id, details, True
)
def test_update_invalid(self):
data = pd.DriverPrivateData(storage=self.fake_storage)
details = ["invalid"]
self.mock_object(self.fake_storage, 'update',
mock.Mock(return_value=True))
self.assertRaises(
ValueError, data.update, self.entity_id, details)
self.assertFalse(self.fake_storage.update.called)
def test_get(self):
data = pd.DriverPrivateData(storage=self.fake_storage)
key = "fake_key"
value = "fake_value"
default_value = "def"
self.mock_object(self.fake_storage, 'get',
mock.Mock(return_value=value))
actual_result = data.get(self.entity_id, key, default_value)
self.assertEqual(value, actual_result)
self.fake_storage.get.assert_called_once_with(
self.entity_id, key, default_value
)
def test_delete(self):
data = pd.DriverPrivateData(storage=self.fake_storage)
key = "fake_key"
self.mock_object(self.fake_storage, 'get',
mock.Mock(return_value=True))
actual_result = data.delete(self.entity_id, key)
self.assertTrue(actual_result)
self.fake_storage.delete.assert_called_once_with(
self.entity_id, key
)
fake_storage_data = {
"entity_id": "fake_id",
"details": {"foo": "bar"},
"context": "fake_context",
"backend_host": "fake_host",
"default": "def",
"delete_existing": True,
"key": "fake_key",
}
def create_arg_list(key_names):
return [fake_storage_data[key] for key in key_names]
def create_arg_dict(key_names):
return dict((key, fake_storage_data[key]) for key in key_names)
@ddt.ddt
class SqlStorageDriverTestCase(test.TestCase):
@ddt.data(
{
"method_name": 'update',
"method_kwargs": create_arg_dict(
["entity_id", "details", "delete_existing"]),
"valid_args": create_arg_list(
["context", "backend_host", "entity_id", "details",
"delete_existing"]
)
},
{
"method_name": 'get',
"method_kwargs": create_arg_dict(["entity_id", "key", "default"]),
"valid_args": create_arg_list(
["context", "backend_host", "entity_id", "key", "default"]),
},
{
"method_name": 'delete',
"method_kwargs": create_arg_dict(["entity_id", "key"]),
"valid_args": create_arg_list(
["context", "backend_host", "entity_id", "key"]),
})
@ddt.unpack
def test_methods(self, method_kwargs, method_name, valid_args):
method = method_name
db_method = 'driver_private_data_' + method_name
with mock.patch('manila.db.api.' + db_method) as db_method:
storage_driver = pd.SqlStorageDriver(
context=fake_storage_data['context'],
backend_host=fake_storage_data['backend_host'])
method = getattr(storage_driver, method)
method(**method_kwargs)
db_method.assert_called_once_with(*valid_args)

View File

@ -28,6 +28,7 @@ from manila import db
from manila.db.sqlalchemy import models
from manila import exception
from manila import quota
from manila.share import drivers_private_data
from manila.share import manager
from manila import test
from manila.tests import utils as test_utils
@ -141,6 +142,23 @@ class ShareManagerTestCase(test.TestCase):
service_ref['id'])
return service_ref
def test_share_manager_instance(self):
fake_service_name = "fake_service"
import_mock = mock.Mock()
self.mock_object(importutils, "import_object", import_mock)
private_data_mock = mock.Mock()
self.mock_object(drivers_private_data, "DriverPrivateData",
private_data_mock)
share_manager = manager.ShareManager(service_name=fake_service_name)
private_data_mock.assert_called_once_with(
context=mock.ANY,
backend_host=share_manager.host,
config_group=fake_service_name
)
self.assertTrue(import_mock.called)
def test_init_host_with_no_shares(self):
self.mock_object(self.share_manager.db, 'share_get_all_by_host',
mock.Mock(return_value=[]))