Provide private data storage API for drivers

- Add implementation of private data storage API
- Add appropriate unit tests

Partially implements bp private-data-storage-api-for-drivers

Change-Id: I61b3cc41f545b540101819feb603e45d0a057a4e
This commit is contained in:
Igor Malinovskiy 2015-05-26 17:53:00 +03:00
parent 94e8c921db
commit 8953627967
10 changed files with 666 additions and 1 deletions

View File

@ -756,3 +756,20 @@ def share_type_extra_specs_update_or_create(context, share_type_id,
return IMPL.share_type_extra_specs_update_or_create(context, return IMPL.share_type_extra_specs_update_or_create(context,
share_type_id, share_type_id,
extra_specs) extra_specs)
def driver_private_data_get(context, host, entity_id, key=None, default=None):
"""Get one, list or all key-value pairs for given host and entity_id."""
return IMPL.driver_private_data_get(context, host, entity_id, key, default)
def driver_private_data_update(context, host, entity_id, details,
delete_existing=False):
"""Update key-value pairs for given host and entity_id."""
return IMPL.driver_private_data_update(context, host, entity_id, details,
delete_existing)
def driver_private_data_delete(context, host, entity_id, key=None):
"""Remove one, list or all key-value pairs for given host and entity_id."""
return IMPL.driver_private_data_delete(context, host, entity_id, key)

View File

@ -0,0 +1,67 @@
# Copyright 2015 Mirantis inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add_driver_private_data_table
Revision ID: 3a482171410f
Revises: 56cdbe267881
Create Date: 2015-04-21 14:47:38.201658
"""
# revision identifiers, used by Alembic.
revision = '3a482171410f'
down_revision = '56cdbe267881'
from alembic import op
from oslo_log import log
import sqlalchemy as sql
from manila.i18n import _LE
LOG = log.getLogger(__name__)
drivers_private_data_table_name = 'drivers_private_data'
def upgrade():
try:
op.create_table(
drivers_private_data_table_name,
sql.Column('created_at', sql.DateTime),
sql.Column('updated_at', sql.DateTime),
sql.Column('deleted_at', sql.DateTime),
sql.Column('deleted', sql.Integer, default=0),
sql.Column('host', sql.String(255),
nullable=False, primary_key=True),
sql.Column('entity_uuid', sql.String(36),
nullable=False, primary_key=True),
sql.Column('key', sql.String(255),
nullable=False, primary_key=True),
sql.Column('value', sql.String(1023), nullable=False),
mysql_engine='InnoDB',
)
except Exception:
LOG.error(_LE("Table |%s| not created!"),
drivers_private_data_table_name)
raise
def downgrade():
try:
op.drop_table(drivers_private_data_table_name)
except Exception:
LOG.error(_LE("%s table not dropped"), drivers_private_data_table_name)
raise

View File

@ -18,6 +18,7 @@
"""Implementation of SQLAlchemy backend.""" """Implementation of SQLAlchemy backend."""
import copy
import datetime import datetime
import sys import sys
import uuid import uuid
@ -2086,6 +2087,97 @@ def share_server_backend_details_get(context, share_server_id,
return dict([(item.key, item.value) for item in query]) return dict([(item.key, item.value) for item in query])
###################
def _driver_private_data_query(session, context, host, entity_id, key=None,
read_deleted=False):
query = model_query(context, models.DriverPrivateData,
session=session, read_deleted=read_deleted)\
.filter_by(host=host)\
.filter_by(entity_uuid=entity_id)
if isinstance(key, list):
return query.filter(models.DriverPrivateData.key.in_(key))
elif key is not None:
return query.filter_by(key=key)
return query
@require_context
def driver_private_data_get(context, host, entity_id, key=None,
default=None, session=None):
if not session:
session = get_session()
query = _driver_private_data_query(session, context, host, entity_id, key)
if key is None or isinstance(key, list):
return dict([(item.key, item.value) for item in query.all()])
else:
result = query.first()
return result["value"] if result is not None else default
@require_context
def driver_private_data_update(context, host, entity_id, details,
delete_existing=False, session=None):
# NOTE(u_glide): following code modifies details dict, that's why we should
# copy it
new_details = copy.deepcopy(details)
if not session:
session = get_session()
with session.begin():
# Process existing data
# NOTE(u_glide): read_deleted=None means here 'read all'
original_data = _driver_private_data_query(
session, context, host, entity_id, read_deleted=None).all()
for data_ref in original_data:
in_new_details = data_ref['key'] in new_details
if in_new_details:
new_value = six.text_type(new_details.pop(data_ref['key']))
data_ref.update({
"value": new_value,
"deleted": 0,
"deleted_at": None
})
data_ref.save(session=session)
elif delete_existing and data_ref['deleted'] != 1:
data_ref.update({
"deleted": 1, "deleted_at": timeutils.utcnow()
})
data_ref.save(session=session)
# Add new data
for key, value in new_details.items():
data_ref = models.DriverPrivateData()
data_ref.update({
"host": host,
"entity_uuid": entity_id,
"key": key,
"value": six.text_type(value)
})
data_ref.save(session=session)
return details
@require_context
def driver_private_data_delete(context, host, entity_id, key=None,
session=None):
if not session:
session = get_session()
with session.begin():
query = _driver_private_data_query(session, context, host,
entity_id, key)
query.update({"deleted": 1, "deleted_at": timeutils.utcnow()})
################### ###################

View File

@ -457,6 +457,15 @@ class NetworkAllocation(BASE, ManilaBase):
default=constants.STATUS_NEW) default=constants.STATUS_NEW)
class DriverPrivateData(BASE, ManilaBase):
"""Represents a private data as key-value pairs for a driver."""
__tablename__ = 'drivers_private_data'
host = Column(String(255), nullable=False, primary_key=True)
entity_uuid = Column(String(36), nullable=False, primary_key=True)
key = Column(String(255), nullable=False, primary_key=True)
value = Column(String(1023), nullable=False)
def register_models(): def register_models():
"""Register Models and create metadata. """Register Models and create metadata.

View File

@ -65,6 +65,7 @@ import manila.share.drivers.netapp.options
import manila.share.drivers.quobyte.quobyte import manila.share.drivers.quobyte.quobyte
import manila.share.drivers.service_instance import manila.share.drivers.service_instance
import manila.share.drivers.zfssa.zfssashare import manila.share.drivers.zfssa.zfssashare
import manila.share.drivers_private_data
import manila.share.manager import manila.share.manager
import manila.volume import manila.volume
import manila.volume.cinder import manila.volume.cinder
@ -114,6 +115,7 @@ _global_opt_lists = [
manila.share.driver.ganesha_opts, manila.share.driver.ganesha_opts,
manila.share.driver.share_opts, manila.share.driver.share_opts,
manila.share.driver.ssh_opts, manila.share.driver.ssh_opts,
manila.share.drivers_private_data.private_data_opts,
manila.share.drivers.emc.driver.EMC_NAS_OPTS, manila.share.drivers.emc.driver.EMC_NAS_OPTS,
manila.share.drivers.emc.plugins.isilon.isilon.ISILON_OPTS, manila.share.drivers.emc.plugins.isilon.isilon.ISILON_OPTS,
manila.share.drivers.generic.share_opts, manila.share.drivers.generic.share_opts,

View File

@ -0,0 +1,176 @@
# Copyright 2015 Mirantis inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Module provides possibility for share drivers to store private information
related to common Manila models like Share or Snapshot.
"""
import abc
from oslo_config import cfg
from oslo_utils import importutils
from oslo_utils import uuidutils
import six
from manila.db import api as db_api
from manila.i18n import _
private_data_opts = [
cfg.StrOpt(
'drivers_private_storage_class',
default='manila.share.drivers_private_data.SqlStorageDriver',
help='The full class name of the Private Data Driver class to use.'),
]
CONF = cfg.CONF
@six.add_metaclass(abc.ABCMeta)
class StorageDriver(object):
def __init__(self, context, backend_host):
# Backend shouldn't access data stored by another backend
self.backend_host = backend_host
self.context = context
@abc.abstractmethod
def get(self, entity_id, key, default):
"""Backend implementation for DriverPrivateData.get() method.
Should return all keys for given 'entity_id' if 'key' is None.
Otherwise should return value for provided 'key'.
If values for provided 'entity_id' or 'key' not found,
should return 'default'.
See DriverPrivateData.get() method for more details.
"""
@abc.abstractmethod
def update(self, entity_id, details, delete_existing):
"""Backend implementation for DriverPrivateData.update() method.
Should update details for given 'entity_id' with behaviour defined
by 'delete_existing' boolean flag.
See DriverPrivateData.update() method for more details.
"""
@abc.abstractmethod
def delete(self, entity_id, key):
"""Backend implementation for DriverPrivateData.delete() method.
Should return delete all keys if 'key' is None.
Otherwise should delete value for provided 'key'.
See DriverPrivateData.update() method for more details.
"""
class SqlStorageDriver(StorageDriver):
def update(self, entity_id, details, delete_existing):
return db_api.driver_private_data_update(
self.context, self.backend_host, entity_id, details,
delete_existing
)
def get(self, entity_id, key, default):
return db_api.driver_private_data_get(
self.context, self.backend_host, entity_id, key, default
)
def delete(self, entity_id, key):
return db_api.driver_private_data_delete(
self.context, self.backend_host, entity_id, key
)
class DriverPrivateData(object):
def __init__(self, storage=None, *args, **kwargs):
"""Init method.
:param storage: None or inheritor of StorageDriver abstract class
:param config_group: Optional -- Config group used for loading settings
:param context: Optional -- Current context
:param backend_host: Optional -- Driver host
"""
config_group_name = kwargs.get('config_group')
CONF.register_opts(private_data_opts, group=config_group_name)
if storage is not None:
self._storage = storage
elif 'context' in kwargs and 'backend_host' in kwargs:
if config_group_name:
conf = getattr(CONF, config_group_name)
else:
conf = CONF
storage_class = conf.drivers_private_storage_class
cls = importutils.import_class(storage_class)
self._storage = cls(kwargs.get('context'),
kwargs.get('backend_host'))
else:
msg = _("You should provide 'storage' parameter or"
" 'context' and 'backend_host' parameters.")
raise ValueError(msg)
def get(self, entity_id, key=None, default=None):
"""Get one, list or all key-value pairs.
:param entity_id: Model UUID
:param key: Key string or list of keys
:param default: Default value for case when key(s) not found
:returns: string or dict
"""
self._validate_entity_id(entity_id)
return self._storage.get(entity_id, key, default)
def update(self, entity_id, details, delete_existing=False):
"""Update or create specified key-value pairs.
:param entity_id: Model UUID
:param details: dict with key-value pairs data. Keys and values should
be strings.
:param delete_existing: boolean flag which determines behaviour
for existing key-value pairs:
True - remove all existing key-value pairs
False (default) - leave as is
"""
self._validate_entity_id(entity_id)
if not isinstance(details, dict):
msg = (_("Provided details %s is not valid dict.")
% six.text_type(details))
raise ValueError(msg)
return self._storage.update(
entity_id, details, delete_existing)
def delete(self, entity_id, key=None):
"""Delete one, list or all key-value pairs.
:param entity_id: Model UUID
:param key: Key string or list of keys
"""
self._validate_entity_id(entity_id)
return self._storage.delete(entity_id, key)
@staticmethod
def _validate_entity_id(entity_id):
if not uuidutils.is_uuid_like(entity_id):
msg = (_("Provided entity_id %s is not valid UUID.")
% six.text_type(entity_id))
raise ValueError(msg)

View File

@ -39,6 +39,7 @@ from manila.i18n import _LW
from manila import manager from manila import manager
from manila import quota from manila import quota
import manila.share.configuration import manila.share.configuration
from manila.share import drivers_private_data
from manila.share import utils as share_utils from manila.share import utils as share_utils
from manila import utils from manila import utils
@ -112,8 +113,16 @@ class ShareManager(manager.SchedulerDependentManager):
msg_args) msg_args)
share_driver = MAPPING[share_driver] share_driver = MAPPING[share_driver]
ctxt = context.get_admin_context()
private_storage = drivers_private_data.DriverPrivateData(
context=ctxt, backend_host=self.host,
config_group=self.configuration.config_group
)
self.driver = importutils.import_object( self.driver = importutils.import_object(
share_driver, configuration=self.configuration) share_driver, private_storage=private_storage,
configuration=self.configuration
)
def _ensure_share_has_pool(self, ctxt, share): def _ensure_share_has_pool(self, ctxt, share):
pool = share_utils.extract_host(share['host'], 'pool') pool = share_utils.extract_host(share['host'], 'pool')

View File

@ -15,11 +15,16 @@
"""Testing of SQLAlchemy backend.""" """Testing of SQLAlchemy backend."""
import ddt
from oslo_utils import uuidutils
import six
from manila import context from manila import context
from manila.db.sqlalchemy import api from manila.db.sqlalchemy import api
from manila import test from manila import test
@ddt.ddt
class SQLAlchemyAPIShareTestCase(test.TestCase): class SQLAlchemyAPIShareTestCase(test.TestCase):
def setUp(self): def setUp(self):
@ -76,3 +81,94 @@ class SQLAlchemyAPIShareTestCase(test.TestCase):
actual_result = api.share_export_locations_get(self.ctxt, share['id']) actual_result = api.share_export_locations_get(self.ctxt, share['id'])
self.assertTrue(actual_result == [initial_location]) self.assertTrue(actual_result == [initial_location])
def _get_driver_test_data(self):
return ("fake@host", uuidutils.generate_uuid())
@ddt.data({"details": {"foo": "bar", "tee": "too"},
"valid": {"foo": "bar", "tee": "too"}},
{"details": {"foo": "bar", "tee": ["test"]},
"valid": {"foo": "bar", "tee": six.text_type(["test"])}})
@ddt.unpack
def test_driver_private_data_update(self, details, valid):
test_host, test_id = self._get_driver_test_data()
initial_data = api.driver_private_data_get(
self.ctxt, test_host, test_id)
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
actual_data = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual({}, initial_data)
self.assertEqual(valid, actual_data)
def test_driver_private_data_update_with_duplicate(self):
test_host, test_id = self._get_driver_test_data()
details = {"tee": "too"}
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
actual_result = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual(details, actual_result)
def test_driver_private_data_update_with_delete_existing(self):
test_host, test_id = self._get_driver_test_data()
details = {"key1": "val1", "key2": "val2", "key3": "val3"}
details_update = {"key1": "val1_upd", "key4": "new_val"}
# Create new details
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
api.driver_private_data_update(self.ctxt, test_host, test_id,
details_update, delete_existing=True)
actual_result = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual(details_update, actual_result)
def test_driver_private_data_get(self):
test_host, test_id = self._get_driver_test_data()
test_key = "foo"
test_keys = [test_key, "tee"]
details = {test_keys[0]: "val", test_keys[1]: "val", "mee": "foo"}
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
actual_result_all = api.driver_private_data_get(
self.ctxt, test_host, test_id)
actual_result_single_key = api.driver_private_data_get(
self.ctxt, test_host, test_id, test_key)
actual_result_list = api.driver_private_data_get(
self.ctxt, test_host, test_id, test_keys)
self.assertEqual(details, actual_result_all)
self.assertEqual(details[test_key], actual_result_single_key)
self.assertEqual(dict.fromkeys(test_keys, "val"), actual_result_list)
def test_driver_private_data_delete_single(self):
test_host, test_id = self._get_driver_test_data()
test_key = "foo"
details = {test_key: "bar", "tee": "too"}
valid_result = {"tee": "too"}
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
api.driver_private_data_delete(self.ctxt, test_host, test_id, test_key)
actual_result = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual(valid_result, actual_result)
def test_driver_private_data_delete_all(self):
test_host, test_id = self._get_driver_test_data()
details = {"foo": "bar", "tee": "too"}
api.driver_private_data_update(self.ctxt, test_host, test_id, details)
api.driver_private_data_delete(self.ctxt, test_host, test_id)
actual_result = api.driver_private_data_get(
self.ctxt, test_host, test_id)
self.assertEqual({}, actual_result)

View File

@ -0,0 +1,179 @@
# Copyright 2015 Mirantis inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ddt
import mock
from oslo_utils import uuidutils
from manila.share import drivers_private_data as pd
from manila import test
@ddt.ddt
class DriverPrivateDataTestCase(test.TestCase):
"""Tests DriverPrivateData."""
def setUp(self):
super(DriverPrivateDataTestCase, self).setUp()
self.fake_storage = mock.Mock()
self.entity_id = uuidutils.generate_uuid()
def test_default_storage_driver(self):
private_data = pd.DriverPrivateData(
storage=None, context="fake", backend_host="fake")
self.assertIsInstance(private_data._storage, pd.SqlStorageDriver)
def test_custom_storage_driver(self):
private_data = pd.DriverPrivateData(storage=self.fake_storage)
self.assertEqual(private_data._storage, self.fake_storage)
def test_invalid_parameters(self):
self.assertRaises(ValueError, pd.DriverPrivateData)
@ddt.data({'context': 'fake'}, {'backend_host': 'fake'})
def test_invalid_single_parameter(self, test_args):
self.assertRaises(ValueError, pd.DriverPrivateData, **test_args)
@ddt.data("111", ["fake"], None)
def test_validate_entity_id_invalid(self, entity_id):
data = pd.DriverPrivateData(storage="fake")
self.assertRaises(ValueError, data._validate_entity_id, entity_id)
def test_validate_entity_id_valid(self):
actual_result = (
pd.DriverPrivateData._validate_entity_id(self.entity_id)
)
self.assertIsNone(actual_result)
def test_update(self):
data = pd.DriverPrivateData(storage=self.fake_storage)
details = {"foo": "bar"}
self.mock_object(self.fake_storage, 'update',
mock.Mock(return_value=True))
actual_result = data.update(
self.entity_id,
details,
delete_existing=True
)
self.assertTrue(actual_result)
self.fake_storage.update.assert_called_once_with(
self.entity_id, details, True
)
def test_update_invalid(self):
data = pd.DriverPrivateData(storage=self.fake_storage)
details = ["invalid"]
self.mock_object(self.fake_storage, 'update',
mock.Mock(return_value=True))
self.assertRaises(
ValueError, data.update, self.entity_id, details)
self.assertFalse(self.fake_storage.update.called)
def test_get(self):
data = pd.DriverPrivateData(storage=self.fake_storage)
key = "fake_key"
value = "fake_value"
default_value = "def"
self.mock_object(self.fake_storage, 'get',
mock.Mock(return_value=value))
actual_result = data.get(self.entity_id, key, default_value)
self.assertEqual(value, actual_result)
self.fake_storage.get.assert_called_once_with(
self.entity_id, key, default_value
)
def test_delete(self):
data = pd.DriverPrivateData(storage=self.fake_storage)
key = "fake_key"
self.mock_object(self.fake_storage, 'get',
mock.Mock(return_value=True))
actual_result = data.delete(self.entity_id, key)
self.assertTrue(actual_result)
self.fake_storage.delete.assert_called_once_with(
self.entity_id, key
)
fake_storage_data = {
"entity_id": "fake_id",
"details": {"foo": "bar"},
"context": "fake_context",
"backend_host": "fake_host",
"default": "def",
"delete_existing": True,
"key": "fake_key",
}
def create_arg_list(key_names):
return [fake_storage_data[key] for key in key_names]
def create_arg_dict(key_names):
return dict((key, fake_storage_data[key]) for key in key_names)
@ddt.ddt
class SqlStorageDriverTestCase(test.TestCase):
@ddt.data(
{
"method_name": 'update',
"method_kwargs": create_arg_dict(
["entity_id", "details", "delete_existing"]),
"valid_args": create_arg_list(
["context", "backend_host", "entity_id", "details",
"delete_existing"]
)
},
{
"method_name": 'get',
"method_kwargs": create_arg_dict(["entity_id", "key", "default"]),
"valid_args": create_arg_list(
["context", "backend_host", "entity_id", "key", "default"]),
},
{
"method_name": 'delete',
"method_kwargs": create_arg_dict(["entity_id", "key"]),
"valid_args": create_arg_list(
["context", "backend_host", "entity_id", "key"]),
})
@ddt.unpack
def test_methods(self, method_kwargs, method_name, valid_args):
method = method_name
db_method = 'driver_private_data_' + method_name
with mock.patch('manila.db.api.' + db_method) as db_method:
storage_driver = pd.SqlStorageDriver(
context=fake_storage_data['context'],
backend_host=fake_storage_data['backend_host'])
method = getattr(storage_driver, method)
method(**method_kwargs)
db_method.assert_called_once_with(*valid_args)

View File

@ -28,6 +28,7 @@ from manila import db
from manila.db.sqlalchemy import models from manila.db.sqlalchemy import models
from manila import exception from manila import exception
from manila import quota from manila import quota
from manila.share import drivers_private_data
from manila.share import manager from manila.share import manager
from manila import test from manila import test
from manila.tests import utils as test_utils from manila.tests import utils as test_utils
@ -141,6 +142,23 @@ class ShareManagerTestCase(test.TestCase):
service_ref['id']) service_ref['id'])
return service_ref return service_ref
def test_share_manager_instance(self):
fake_service_name = "fake_service"
import_mock = mock.Mock()
self.mock_object(importutils, "import_object", import_mock)
private_data_mock = mock.Mock()
self.mock_object(drivers_private_data, "DriverPrivateData",
private_data_mock)
share_manager = manager.ShareManager(service_name=fake_service_name)
private_data_mock.assert_called_once_with(
context=mock.ANY,
backend_host=share_manager.host,
config_group=fake_service_name
)
self.assertTrue(import_mock.called)
def test_init_host_with_no_shares(self): def test_init_host_with_no_shares(self):
self.mock_object(self.share_manager.db, 'share_get_all_by_host', self.mock_object(self.share_manager.db, 'share_get_all_by_host',
mock.Mock(return_value=[])) mock.Mock(return_value=[]))