openstack/karbor
Code Issues Proposed changes

Implement Provider and ProviderRegistry

Browse Source 
Change-Id: I0898a82e760098a168922c15fe035d65f0df7a43
Closes-Bug: #1546851
This commit is contained in:
wangliuan 2016-02-18 11:20:21 +08:00 committed by Yuval Brik
parent 90d19c48c7
commit 00b2e6649c
7 changed files with 258 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
setup.cfg
View File

@ -37,6 +37,11 @@ console_scripts =
smaug-protection = smaug.cmd.protection:main
smaug.database.migration_backend =
sqlalchemy = oslo_db.sqlalchemy.migration
smaug.protections =
smaug-swift-bank-plugin = smaug.services.protection.plugins.swift_bank_plugin:SwiftBankPlugin
smaug-volume-protection-plugin = smaug.services.protection.plugins.cinder_backup_plugin:CinderBackupPlugin
smaug.provider =
provider-registry = smaug.services.protection.provider:ProviderRegistry
smaug.protectables =
project = smaug.services.protection.protectable_plugins.project:ProjectProtectablePlugin

3
smaug/common/constants.py
View File

@ -17,6 +17,9 @@ OPERATION_START = 'start'
OPERATION_DELETE = 'delete'
OPERATION_SUSPEND = 'suspend'
# plugin type
PLUGIN_BANK = 'bank'
# supported resource types
RESOURCE_TYPES = (PROJECT_RESOURCE_TYPE,
SERVER_RESOURCE_TYPE,

7
smaug/services/protection/checkpoint.py
View File

@ -35,11 +35,10 @@ class CheckpointSerializer(object):
class CheckpointCollection(object):
def __init__(self):
def __init__(self, bank_plugin):
super(CheckpointCollection, self).__init__()
self.checkpoint_serializer = None
self.bank_plugin = None
# TODO(wangliuan)
self.checkpoint_serializer = CheckpointSerializer()
self._bank_plugin = bank_plugin
def list(self, list_options):
# TODO(wangliuan)

96
smaug/services/protection/manager.py
View File

@ -1,4 +1,4 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
@ -19,25 +19,31 @@ import six
from oslo_config import cfg
from oslo_log import log as logging
import oslo_messaging as messaging
from oslo_utils import importutils
from smaug.i18n import _LI, _LE
from smaug import exception
from smaug.i18n import _LI, _LE
from smaug import manager
from smaug.resource import Resource
from smaug.services.protection import protectable_registry as p_reg
from smaug.services.protection.provider import PluggableProtectionProvider
from smaug import utils
LOG = logging.getLogger(__name__)
protection_manager_opts = [
cfg.IntOpt('update_protection_stats_interval',
default=3600,
help='update protection status interval')
help='update protection status interval'),
cfg.StrOpt('provider_registry',
default='smaug.services.protection.provider.ProviderRegistry',
help='the provider registry')
]
CONF = cfg.CONF
CONF.register_opts(protection_manager_opts)
PROVIDER_NAMESPACE = 'smaug.provider'
class ProtectionManager(manager.Manager):
"""Smaug Protection Manager."""
@ -49,9 +55,9 @@ class ProtectionManager(manager.Manager):
def __init__(self, service_name=None,
*args, **kwargs):
super(ProtectionManager, self).__init__(*args, **kwargs)
# TODO(wangliuan) more params and use profiler.trace_cls
self.provider_registry = importutils.import_object(
'smaug.services.protection.provider.ProviderRegistry')
provider_reg = CONF.provider_registry
self.provider_registry = utils.load_plugin(PROVIDER_NAMESPACE,
provider_reg)
self.flow_engine = None
# TODO(wangliuan)
@ -67,7 +73,7 @@ class ProtectionManager(manager.Manager):
:param plan: Define that protection plan should be done
"""
LOG.info(_LI("Starting protection service:protect action"))
LOG.debug('restoration :%s tpye:%s', plan,
LOG.debug('protecting :%s tpye:%s', plan,
type(plan))
# TODO(wangliuan)
@ -108,7 +114,6 @@ class ProtectionManager(manager.Manager):
def list_checkpoints(self, context, provider_id, marker=None, limit=None,
sort_keys=None, sort_dirs=None, filters=None):
# TODO(wangliuan)
LOG.info(_LI("Starting list checkpoints. "
"provider_id:%s"), provider_id)
@ -173,64 +178,6 @@ class ProtectionManager(manager.Manager):
# TODO(wangliuan)
pass
def list_providers(self, context, marker=None, limit=None, sort_keys=None,
sort_dirs=None, filters=None):
# TODO(wangliuan)
LOG.info(_LI("Starting list providers. "
"filters:%s"), filters)
return_stub = [
{
"id": "2220f8b1-975d-4621-a872-fa9afb43cb6c",
"name": "OS Infra Provider",
"description": "This provider uses OpenStack's"
" own services (swift, cinder) as storage",
"extended_info_schema": {
"OS::Nova::Cinder": {
"type": "object",
"properties": {
"use_cbt": {
"type": "boolean",
"title": "Use CBT",
"description":
"Use Changed Block"
" Tracking when backin up this volume"
}
}
}
}
}
]
return return_stub
def show_provider(self, context, provider_id):
# TODO(wangliuan)
LOG.info(_LI("Starting show provider. "
"provider_id:%s"), provider_id)
return_stub = {
"id": "2220f8b1-975d-4621-a872-fa9afb43cb6c",
"name": "OS Infra Provider",
"description": "This provider uses OpenStack's"
"own services (swift, cinder) as storage",
"extended_info_schema": {
"OS::Nova::Cinder": {
"type": "object",
"properties": {
"use_cbt": {
"type": "boolean",
"title": "Use CBT",
"description": "Use Changed"
" Block Tracking"
" when backin up"
" this volume"
}
}
}
}
}
return return_stub
def list_protectable_types(self, context):
LOG.info(_LI("Start to list protectable types."))
return p_reg.ProtectableRegistry.list_resource_types()
@ -305,3 +252,18 @@ class ProtectionManager(manager.Manager):
result.append(dict(type=resource.type, id=resource.id))
return result
def list_providers(self, list_option=None):
return self.provider_registry.list_providers(list_option)
def show_provider(self, provider_id):
provider = self.provider_registry.show_provider(provider_id)
if isinstance(provider, PluggableProtectionProvider):
response = {'id': provider.id,
'name': provider.name,
'description': provider.description,
'extended_info_schema': provider.extended_info_schema,
}
return response
else:
raise exception.ProviderNotFound(provider_id=provider_id)

140
smaug/services/protection/provider.py
View File

@ -1,5 +1,5 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
@ -13,30 +13,91 @@
from oslo_config import cfg
from oslo_log import log as logging
from smaug.common import constants
from smaug.i18n import _LE
from smaug.services.protection import checkpoint
from smaug import utils
provider_opt = [
cfg.MultiStrOpt('plugin',
default='',
help='plugins to use for protection'),
cfg.StrOpt('description',
default='',
help='the description of provider'),
cfg.StrOpt('provider_id',
default='',
help='the provider id')
]
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
PROTECTION_NAMESPACE = 'smaug.protections'
class PluggableProtectionProvider(object):
def __init__(self):
def __init__(self, provider_id, provider_name, description, plugins):
super(PluggableProtectionProvider, self).__init__()
self._id = provider_id
self._name = provider_name
self._description = description
self._extended_info_schema = {'options_schema': {},
'restore_schema': {},
'saved_info_schema': {}}
self.checkpoint_collection = None
self._bank_plugin = None
self._plugin_map = {}
self.checkpoint_collection = None
# TODO(wangliuan)
def _load_plugins(self, cfg_file):
# TODO(wangliuan)
pass
self._load_plugins(plugins=plugins)
if self._bank_plugin:
self.checkpoint_collection = checkpoint.CheckpointCollection(
self._bank_plugin)
else:
LOG.error(_LE('Bank plugin not exist,check your configuration'))
@property
def id(self):
return self._id
@property
def name(self):
return self._name
@property
def description(self):
return self._description
@property
def extended_info_schema(self):
return self._extended_info_schema
def _load_plugins(self, plugins):
for plugin_name in plugins:
try:
plugin = utils.load_plugin(PROTECTION_NAMESPACE, plugin_name)
except Exception:
LOG.exception(_LE("Load protection plugin: %s failed."),
plugin_name)
raise
else:
self._plugin_map[plugin_name] = plugin
if constants.PLUGIN_BANK in plugin_name.lower():
self._bank_plugin = plugin
if hasattr(plugin, 'get_options_schema'):
self._extended_info_schema['options_schema'][plugin_name] \
= plugin.get_options_schema()
if hasattr(plugin, 'get_restore_schema'):
self._extended_info_schema['restore_schema'][plugin_name] \
= plugin.get_restore_schema()
if hasattr(plugin, 'get_saved_info_schema'):
self._extended_info_schema['saved_info_schema'][plugin_name] \
= plugin.get_saved_info_schema()
def get_checkpoint_collection(self):
# TODO(wangliuan)
pass
return self.checkpoint_collection
def build_task_flow(self, plan):
def build_task_flow(self, ctx):
# TODO(wangliuan)
pass
@ -44,16 +105,55 @@ class PluggableProtectionProvider(object):
class ProviderRegistry(object):
def __init__(self):
super(ProviderRegistry, self).__init__()
# TODO(wangliuan)
self.providers = {}
self._load_providers()
def load_providers(self, cfg_file):
# TODO(wangliuan)
pass
def _load_providers(self):
"""load provider
def list_providers(self, list_option):
# TODO(wangliuan)
pass
smaug.conf example:
[default]
enabled_providers=provider1,provider2
[provider1]
provider_id='' configured by admin
plugin=BANK define in setup.cfg
plugin=VolumeProtectionPlugin define in setup.cfg
description='the description of provider1'
[provider2]
provider_id='' configured by admin
plugin=BANK define in setup.cfg
plugin=VolumeProtectionPlugin define in setup.cfg
plugin=ServerProtectionPlugin define in setup.cfg
description='the description of provider2'
"""
if CONF.enabled_providers:
for provider_name in CONF.enabled_providers:
CONF.register_opts(provider_opt, group=provider_name)
plugins = getattr(CONF, provider_name).plugin
description = getattr(CONF, provider_name).description
provider_id = getattr(CONF, provider_name).provider_id
if not all([plugins, provider_id]):
LOG.error(_LE("Invalid provider:%s,check provider"
" configuration"),
provider_name)
continue
try:
provider = PluggableProtectionProvider(provider_id,
provider_name,
description,
plugins)
except Exception:
LOG.exception(_LE("Load provider: %s failed."),
provider_name)
else:
self.providers[provider_id] = provider
def list_providers(self, list_option=None):
if not list_option:
return [dict(id=provider.id, name=provider.name,
description=provider.description)
for provider in self.providers.values()]
# It seems that we don't need list_option
def show_provider(self, provider_id):
# TODO(wangliuan)
pass
return self.providers.get(provider_id, None)

78
smaug/tests/unit/protection/test_provider.py Normal file
View File

@ -0,0 +1,78 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from oslo_config import cfg
from smaug.services.protection import provider
from smaug.tests import base
provider_opt = [
cfg.MultiStrOpt('plugin',
default='',
help='plugins to use for protection'),
cfg.StrOpt('description',
default='',
help='the description of provider'),
cfg.StrOpt('provider_id',
default='',
help='the provider id')
]
CONF = cfg.CONF
CONF.register_opt(cfg.ListOpt('enabled_providers',
default=['provider1', 'provider2']))
CONF.register_opts(provider_opt, group='provider1')
CONF.register_opts(provider_opt, group='provider2')
class ProviderRegistryTest(base.TestCase):
def setUp(self):
super(ProviderRegistryTest, self).setUp()
CONF.set_override('plugin', ['SERVER', 'VOLUME'],
group='provider1')
CONF.set_override('plugin', ['SERVER'],
group='provider2')
CONF.set_override('description', 'FAKE1', group='provider1')
CONF.set_override('description', 'FAKE2', group='provider2')
CONF.set_override('provider_id', 'id1', group='provider1')
CONF.set_override('provider_id', 'id2', group='provider2')
@mock.patch.object(provider.PluggableProtectionProvider, '_load_plugins')
def test_load_providers(self, mock_load_plugins):
CONF.set_override('plugin', ['SERVER'],
group='provider2')
pr = provider.ProviderRegistry()
self.assertTrue(mock_load_plugins.called)
self.assertEqual(len(pr.providers), 2)
@mock.patch.object(provider.PluggableProtectionProvider, '_load_plugins')
def test_load_providers_with_no_plugins(self, mock_load_plugins):
CONF.set_override('plugin', None,
group='provider2')
pr = provider.ProviderRegistry()
self.assertEqual(mock_load_plugins.call_count, 1)
self.assertEqual(len(pr.providers), 1)
@mock.patch.object(provider.PluggableProtectionProvider, '_load_plugins')
def test_list_provider(self, mock_load_plugins):
CONF.set_override('plugin', ['SERVER'],
group='provider2')
pr = provider.ProviderRegistry()
self.assertEqual(2, len(pr.list_providers()))
@mock.patch.object(provider.PluggableProtectionProvider, '_load_plugins')
def test_show_provider(self, mock_load_plugins):
CONF.set_override('plugin', ['SERVER'],
group='provider2')
pr = provider.ProviderRegistry()
provider_list = pr.list_providers()
for provider_node in provider_list:
self.assertTrue(pr.show_provider(provider_node['id']))

22
smaug/utils.py
View File

@ -1,4 +1,4 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
@ -16,13 +16,15 @@ import os
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import importutils
from oslo_utils import strutils
from oslo_utils import timeutils
import six
from smaug import exception
from smaug.i18n import _
from smaug.i18n import _, _LE
from stevedore import driver
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -121,3 +123,19 @@ def get_bool_param(param_string, params):
raise exception.InvalidParameterValue(err=msg)
return strutils.bool_from_string(param, strict=True)
def load_plugin(namespace, plugin_name):
try:
# Try to resolve plugin by name
mgr = driver.DriverManager(namespace, plugin_name)
plugin_class = mgr.driver
except RuntimeError as e1:
# fallback to class name
try:
plugin_class = importutils.import_class(plugin_name)
except ImportError as e2:
LOG.exception(_LE("Error loading plugin by name, %s"), e1)
LOG.exception(_LE("Error loading plugin by class, %s"), e2)
raise ImportError(_("Class not found."))
return plugin_class()