openstack/cinderlib
Code Issues Proposed changes

Add metadata plugin mechanism

Browse Source 
Until now we only supported storing resources in memory, and that
mechanism was tightly coupled with everything else.

This patch decouples the metadata storing into a plugin mechanism using
'Python entrypoints.  The default storing mechanism is still in memory,
and for now it's the only mechanism available.

The JSON serialization mechanism has not yet been adapted to the
metadata plugin mechanism, so it will only work with the default plugin.
This commit is contained in:
Gorka Eguileor
2018-03-15 18:23:19 +01:00
parent db5ed94fa3
commit d913aedd3d
10 changed files with 1176 additions and 793 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.rst
View File

@@ -10,6 +10,7 @@ History
- Modify fields on connect method.
- Support setting custom root_helper.
- Setting default project_id and user_id.
- Persistence plugin mechanism
- Bug fixes:

12
cinderlib/__init__.py
View File

@@ -1,8 +1,20 @@
from __future__ import absolute_import
import cinderlib.cinderlib as clib
from cinderlib.cinderlib import * # noqa
from cinderlib import serialization
from cinderlib import objects
__author__ = """Gorka Eguileor"""
__email__ = 'geguileo@redhat.com'
__version__ = '0.1.0'
__all__ = clib.__all__
DEFAULT_PROJECT_ID = objects.DEFAULT_PROJECT_ID
DEFAULT_USER_ID = objects.DEFAULT_USER_ID
Volume = objects.Volume
Snapshot = objects.Snapshot
Connection = objects.Connection
load = serialization.load
json = serialization.json
jsons = serialization.jsons

837
cinderlib/cinderlib.py
View File

@@ -14,13 +14,9 @@
# under the License.
from __future__ import absolute_import
import collections
import functools
import json as json_lib
import logging
import os
import requests
import uuid
from cinder import coordination
# NOTE(geguileo): If we want to prevent eventlet from monkey_patching we would
@@ -28,42 +24,18 @@ from cinder import coordination
# NOTE(geguileo): Probably a good idea not to depend on cinder.cmd.volume
# having all the other imports as they could change.
from cinder.cmd import volume as volume_cmd
from cinder import context
from cinder import exception
from cinder.objects import base as cinder_base_ovo
from cinder import utils
from cinder.volume import configuration
from oslo_utils import importutils
from oslo_versionedobjects import base as base_ovo
from os_brick import exception as brick_exception
from os_brick.privileged import rootwrap
import six
import requests
from cinderlib import objects
from cinderlib import persistence
from cinderlib import serialization
__all__ = ['setup', 'load', 'json', 'jsons', 'Backend', 'Volume', 'Snapshot',
'Connection', 'DEFAULT_PROJECT_ID', 'DEFAULT_USER_ID']
DEFAULT_PROJECT_ID = 'cinderlib'
DEFAULT_USER_ID = 'cinderlib'
CONTEXT = context.RequestContext(user_id=DEFAULT_USER_ID,
project_id=DEFAULT_PROJECT_ID,
is_admin=True,
overwrite=False)
volume_cmd.objects.register_all()
def set_context(project_id=None, user_id=None):
global CONTEXT
project_id = project_id or DEFAULT_PROJECT_ID
user_id = user_id or DEFAULT_USER_ID
if project_id != CONTEXT.project_id or user_id != CONTEXT.user_id:
CONTEXT.user_id = user_id
CONTEXT.project_id = project_id
Volume.DEFAULT_FIELDS_VALUES['user_id'] = user_id
Volume.DEFAULT_FIELDS_VALUES['project_id'] = project_id
__all__ = ['setup', 'Backend']
class Backend(object):
@@ -93,16 +65,15 @@ class Backend(object):
self.driver = importutils.import_object(
conf.volume_driver,
configuration=conf,
db=OVO.fake_db,
db=self.persistence.db,
host=volume_cmd.CONF.host,
cluster_name=None, # No clusters for now: volume_cmd.CONF.cluster,
active_backend_id=None) # No failover for now
self.driver.do_setup(CONTEXT)
self.driver.do_setup(objects.CONTEXT)
self.driver.check_for_setup_error()
self.driver.init_capabilities()
self.driver.set_throttle()
self.driver.set_initialized()
self.volumes = set()
self._driver_cfg = driver_cfg
def __repr__(self):
@@ -116,44 +87,13 @@ class Backend(object):
return self._driver_cfg['volume_backend_name']
@property
def config(self):
if self.output_all_backend_info:
return self._driver_cfg
return {'volume_backend_name': self._driver_cfg['volume_backend_name']}
def volumes(self):
return self.persistence.get_volumes(backend_name=self.id)
@property
def json(self):
result = [volume.json for volume in self.volumes]
# We only need to output the full backend configuration once
if self.output_all_backend_info:
backend = {'volume_backend_name': self.id}
for volume in result:
volume['backend'] = backend
return {'class': type(self).__name__,
'backend': self.config,
'volumes': result}
@property
def jsons(self):
return json_lib.dumps(self.json)
@classmethod
def load(cls, json_src):
backend = Backend.load_backend(json_src['backend'])
for volume in json_src['volumes']:
Volume.load(volume)
return backend
@classmethod
def load_backend(cls, backend_data):
backend_name = backend_data['volume_backend_name']
if backend_name in cls.backends:
return cls.backends[backend_name]
if len(backend_data) > 1:
return cls(**backend_data)
raise Exception('Backend not present in system or json.')
def volumes_filtered(self, volume_id=None, volume_name=None):
return self.persistence.get_volumes(backend_name=self.id,
volume_id=volume_id,
volume_name=volume_name)
def stats(self, refresh=False):
stats = self.driver.get_volume_stats(refresh=refresh)
@@ -161,8 +101,9 @@ class Backend(object):
def create_volume(self, size, name='', description='', bootable=False,
**kwargs):
vol = Volume(self, size=size, name=name, description=description,
bootable=bootable, **kwargs)
vol = objects.Volume(self, size=size, name=name,
description=description, bootable=bootable,
availability_zone=self.id, **kwargs)
vol.create()
return vol
@@ -174,13 +115,12 @@ class Backend(object):
def global_setup(cls, file_locks_path=None, root_helper='sudo',
suppress_requests_ssl_warnings=True, disable_logs=True,
non_uuid_ids=False, output_all_backend_info=False,
project_id=None, user_id=None, **log_params):
project_id=None, user_id=None, persistence_config=None,
**log_params):
# Global setup can only be set once
if cls.global_initialization:
raise Exception('Already setup')
set_context(project_id, user_id)
# Prevent driver dynamic loading clearing configuration options
volume_cmd.CONF._ConfigOpts__cache = MyDict()
@@ -189,7 +129,10 @@ class Backend(object):
configuration.cfg.StrOpt('stateless_cinder'),
group=configuration.SHARED_CONF_GROUP)
OVO._ovo_init(non_uuid_ids)
cls.persistence = persistence.setup(persistence_config)
serialization.setup(cls)
objects.setup(cls.persistence, Backend, project_id, user_id,
non_uuid_ids)
cls._set_logging(disable_logs, **log_params)
cls._set_priv_helper(root_helper)
@@ -277,730 +220,48 @@ class Backend(object):
volume_cmd.CONF.coordination.backend_url = 'file://' + file_locks_path
coordination.COORDINATOR.start()
setup = Backend.global_setup
def load(json_src):
"""Load any json serialized cinderlib object."""
if isinstance(json_src, six.string_types):
json_src = json_lib.loads(json_src)
if isinstance(json_src, list):
return [globals()[obj['class']].load(obj) for obj in json_src]
return globals()[json_src['class']].load(json_src)
def json():
"""Conver to Json everything we have in this system."""
return [backend.json for backend in Backend.backends.values()]
def jsons():
"""Convert to a Json string everything we have in this system."""
return json_lib.dumps(json())
class Object(object):
"""Base class for our resource representation objects."""
DEFAULT_FIELDS_VALUES = {}
objects = collections.defaultdict(dict)
def __init__(self, backend, **fields_data):
self.backend = backend
__ovo = fields_data.get('__ovo')
if __ovo:
self._ovo = __ovo
else:
self._ovo = self._create_ovo(**fields_data)
cls = type(self)
cls.objects = Object.objects[cls.__name__]
# TODO: Don't replace if present is newer
self.objects[self._ovo.id] = self
def _to_primitive(self):
return None
def _create_ovo(self, **fields_data):
# The base are the default values we define on our own classes
fields_values = self.DEFAULT_FIELDS_VALUES.copy()
# Apply the values defined by the caller
fields_values.update(fields_data)
# We support manually setting the id, so set only if not already set
# or if set to None
if not fields_values.get('id'):
fields_values['id'] = self.new_uuid()
# Set non set field values based on OVO's default value and on whether
# it is nullable or not.
for field_name, field in self.OVO_CLASS.fields.items():
if field.default != cinder_base_ovo.fields.UnspecifiedDefault:
fields_values.setdefault(field_name, field.default)
elif field.nullable:
fields_values.setdefault(field_name, None)
return self.OVO_CLASS(context=CONTEXT, **fields_values)
@property
def config(self):
if self.output_all_backend_info:
return self._driver_cfg
return {'volume_backend_name': self._driver_cfg['volume_backend_name']}
@property
def json(self):
ovo = self._ovo.obj_to_primitive()
result = [volume.json for volume in self.volumes]
# We only need to output the full backend configuration once
if self.output_all_backend_info:
backend = {'volume_backend_name': self.id}
for volume in result:
volume['backend'] = backend
return {'class': type(self).__name__,
'backend': self.backend.config,
'ovo': ovo}
'backend': self.config,
'volumes': result}
@property
def jsons(self):
return json_lib.dumps(self.json)
def __repr__(self):
return ('<cinderlib.%s object %s on backend %s>' %
(type(self).__name__,
self.id,
self.backend.id))
@classmethod
def load(cls, json_src):
backend = Backend.load_backend(json_src['backend'])
for volume in json_src['volumes']:
objects.Volume.load(volume)
return backend
@classmethod
def load_backend(cls, backend_data):
backend_name = backend_data['volume_backend_name']
if backend_name in cls.backends:
return cls.backends[backend_name]
if len(backend_data) > 1:
return cls(**backend_data)
backend_name = json_src['backend']['volume_backend_name']
if backend_name in Backend.backends:
backend = Backend.backends[backend_name]
elif len(json_src['backend']) == 1:
raise Exception('Backend not present in system or json.')
else:
backend = Backend(**json_src['backend'])
ovo = cinder_base_ovo.CinderObject.obj_from_primitive(json_src['ovo'],
CONTEXT)
return cls._load(backend, ovo)
def _replace_ovo(self, ovo):
self._ovo = ovo
@staticmethod
def new_uuid():
return str(uuid.uuid4())
def __getattr__(self, name):
if name == '_ovo':
raise AttributeError('Attribute _ovo is not yet set')
return getattr(self._ovo, name)
class Volume(Object):
OVO_CLASS = volume_cmd.objects.Volume
DEFAULT_FIELDS_VALUES = {
'size': 1,
'user_id': CONTEXT.user_id,
'project_id': CONTEXT.project_id,
'host': volume_cmd.CONF.host,
'status': 'creating',
'attach_status': 'detached',
'metadata': {},
'admin_metadata': {},
'glance_metadata': {},
}
_ignore_keys = ('id', 'volume_attachment', 'snapshots')
def __init__(self, backend_or_vol, **kwargs):
# Accept a volume as additional source data
if isinstance(backend_or_vol, Volume):
for key in backend_or_vol._ovo.fields:
if (backend_or_vol._ovo.obj_attr_is_set(key) and
key not in self._ignore_keys):
kwargs.setdefault(key, getattr(backend_or_vol._ovo, key))
backend_or_vol = backend_or_vol.backend
if '__ovo' not in kwargs:
if 'description' in kwargs:
kwargs['display_description'] = kwargs.pop('description')
if 'name' in kwargs:
kwargs['display_name'] = kwargs.pop('name')
kwargs.setdefault(
'volume_attachment',
volume_cmd.objects.VolumeAttachmentList(context=CONTEXT))
kwargs.setdefault(
'snapshots',
volume_cmd.objects.SnapshotList(context=CONTEXT))
super(Volume, self).__init__(backend_or_vol, **kwargs)
self.snapshots = set()
self.connections = []
self._populate_data()
self.backend.volumes.add(self)
def _to_primitive(self):
local_attach = self.local_attach.id if self.local_attach else None
return {'local_attach': local_attach,
'exported': self.exported}
@classmethod
def _load(cls, backend, ovo):
# Restore snapshot's circular reference removed on serialization
for snap in ovo.snapshots:
snap.volume = ovo
# If this object is already present it will be replaced
obj = Object.objects['Volume'].get(ovo.id)
if obj:
obj._replace_ovo(ovo)
else:
obj = cls(backend, __ovo=ovo)
return obj
def _replace_ovo(self, ovo):
super(Volume, self)._replace_ovo(ovo)
self._populate_data()
def _populate_data(self):
old_snapshots = {snap.id: snap for snap in self.snapshots}
for snap_ovo in self._ovo.snapshots:
snap = Object.objects['Snapshot'].get(snap_ovo.id)
if snap:
snap._replace_ovo(snap_ovo)
del old_snapshots[snap.id]
else:
snap = Snapshot(self, __ovo=snap_ovo)
self.snapshots.add(snap)
for snap_id, snap in old_snapshots.items():
self.snapshots.discard(snap)
# We leave snapshots in the global DB just in case...
# del Object.objects['Snapshot'][snap_id]
old_connections = {conn.id: conn for conn in self.connections}
for conn_ovo in self._ovo.volume_attachment:
conn = Object.objects['Connection'].get(conn_ovo.id)
if conn:
conn._replace_ovo(conn_ovo)
del old_connections[conn.id]
else:
conn = Connection(self.backend, volume=self, __ovo=conn_ovo)
self.connections.append(conn)
for conn_id, conn in old_connections.items():
self.connections.remove(conn)
# We leave connections in the global DB just in case...
# del Object.objects['Connection'][conn_id]
data = getattr(self._ovo, 'cinderlib_data', {})
self.exported = data.get('exported', False)
self.local_attach = data.get('local_attach', None)
if self.local_attach:
self.local_attach = Object.objects['Connection'][self.local_attach]
def create(self):
try:
model_update = self.backend.driver.create_volume(self._ovo)
self._ovo.status = 'available'
if model_update:
self._ovo.update(model_update)
except Exception:
self._ovo.status = 'error'
# TODO: raise with the vol info
raise
def delete(self):
# Some backends delete existing snapshots while others leave them
try:
self.backend.driver.delete_volume(self._ovo)
self._ovo.status = 'deleted'
self._ovo.deleted = True
# volume.deleted_at =
except Exception:
self._ovo.status = 'error'
# TODO: raise with the vol info
raise
self.backend.volumes.discard(self)
def extend(self, size):
volume = self._ovo
volume.previous_status = volume.status
volume.status = 'extending'
try:
self.backend.driver.extend_volume(volume, size)
except Exception:
volume.status = 'error'
# TODO: raise with the vol info
raise
volume.size = size
volume.status = volume.previous_status
volume.previous_status = None
def clone(self, **new_vol_attrs):
new_vol_attrs['source_vol_id'] = self.id
new_vol = Volume(self, **new_vol_attrs)
try:
model_update = self.backend.driver.create_cloned_volume(
new_vol._ovo, self._ovo)
new_vol.status = 'available'
if model_update:
new_vol.update(model_update)
except Exception:
new_vol.status = 'error'
# TODO: raise with the new volume info
raise
return new_vol
def create_snapshot(self, name='', description='', **kwargs):
snap = Snapshot(self, name=name, description=description, **kwargs)
snap.create()
self.snapshots.add(snap)
self._ovo.snapshots.objects.append(snap._ovo)
return snap
def attach(self):
connector_dict = utils.brick_get_connector_properties(
self.backend.configuration.use_multipath_for_image_xfer,
self.backend.configuration.enforce_multipath_for_image_xfer)
conn = self.connect(connector_dict)
try:
conn.attach()
except Exception:
self.disconnect(conn)
raise
return conn
def detach(self, force=False, ignore_errors=False):
if not self.local_attach:
raise Exception('Not attached')
exc = brick_exception.ExceptionChainer()
conn = self.local_attach
try:
conn.detach(force, ignore_errors, exc)
except Exception:
if not force:
raise
with exc.context(force, 'Unable to disconnect'):
conn.disconnect(force)
if exc and not ignore_errors:
raise exc
def connect(self, connector_dict, **ovo_fields):
if not self.exported:
model_update = self.backend.driver.create_export(CONTEXT,
self._ovo,
connector_dict)
if model_update:
self._ovo.update(model_update)
self.exported = True
try:
conn = Connection.connect(self, connector_dict, **ovo_fields)
self.connections.append(conn)
self._ovo.status = 'in-use'
except Exception:
if not self.connections:
self._remove_export()
# TODO: Improve raised exception
raise
return conn
def _disconnect(self, connection):
self.connections.remove(connection)
if not self.connections:
self._remove_export()
self._ovo.status = 'available'
def disconnect(self, connection, force=False):
connection._disconnect(force)
self._disconnect(connection)
def cleanup(self):
for attach in self.attachments:
attach.detach()
self._remove_export()
def _remove_export(self):
self.backend.driver.remove_export(self._context, self._ovo)
self.exported = False
class Connection(Object):
OVO_CLASS = volume_cmd.objects.VolumeAttachment
@classmethod
def connect(cls, volume, connector, *kwargs):
conn_info = volume.backend.driver.initialize_connection(
volume._ovo, connector)
conn = cls(volume.backend,
connector=connector,
volume=volume,
status='attached',
attach_mode='rw',
connection_info=conn_info,
*kwargs)
volume._ovo.volume_attachment.objects.append(conn._ovo)
return conn
def __init__(self, *args, **kwargs):
self.connected = True
self.volume = kwargs.pop('volume')
self.connector = kwargs.pop('connector', None)
self.attach_info = kwargs.pop('attach_info', None)
if '__ovo' not in kwargs:
kwargs['volume'] = self.volume._ovo
kwargs['volume_id'] = self.volume._ovo.id
super(Connection, self).__init__(*args, **kwargs)
self._populate_data()
def _to_primitive(self):
result = {
'connector': self.connector,
}
if self.attach_info:
attach_info = self.attach_info.copy()
connector = attach_info['connector']
attach_info['connector'] = {
'use_multipath': connector.use_multipath,
'device_scan_attempts': connector.device_scan_attempts,
}
else:
attach_info = None
result['attachment'] = attach_info
return result
def _populate_data(self):
# Ensure circular reference is set
self._ovo.volume = self.volume._ovo
data = getattr(self._ovo, 'cinderlib_data', None)
if data:
self.connector = data.get('connector', None)
self.attach_info = data.get('attachment', None)
conn = (self.attach_info or {}).get('connector')
if isinstance(conn, dict):
self.attach_info['connector'] = utils.brick_get_connector(
self.connection_info['driver_volume_type'],
conn=self.connection_info,
**conn)
self.attached = bool(self.attach_info)
def _replace_ovo(self, ovo):
super(Connection, self)._replace_ovo(ovo)
self._populate_data()
@classmethod
def _load(cls, backend, ovo):
# Turn this around and do a Volume load
volume = ovo.volume
volume.volume_attachment.objects.append(ovo)
# Remove circular reference
delattr(ovo, base_ovo._get_attrname('volume'))
Volume._load(backend, volume)
return Connection.objects[ovo.id]
def _disconnect(self, force=False):
self.backend.driver.terminate_connection(self._ovo.volume,
self.connector,
force=force)
self.connected = False
self._ovo.volume.volume_attachment.objects.remove(self._ovo)
self._ovo.status = 'detached'
self._ovo.deleted = True
def disconnect(self, force=False):
self._disconnect(force)
self.volume._disconnect(self)
def attach(self):
self.attach_info = self.backend.driver._connect_device(
self.connection_info)
self.attached = True
self.volume.local_attach = self
def detach(self, force=False, ignore_errors=False, exc=None):
if not exc:
exc = brick_exception.ExceptionChainer()
connector = self.attach_info['connector']
with exc.context(force, 'Disconnect failed'):
connector.disconnect_volume(self.connection_info['data'],
self.attach_info['device'],
force=force,
ignore_errors=ignore_errors)
self.attached = False
self.volume.local_attach = None
if exc and not ignore_errors:
raise exc
@property
def path(self):
if self.attach_info:
return self.attach_info['device']['path']
return None
class Snapshot(Object):
OVO_CLASS = volume_cmd.objects.Snapshot
DEFAULT_FIELDS_VALUES = {
'status': 'creating',
'metadata': {},
}
def __init__(self, volume, **kwargs):
self.volume = volume
if '__ovo' in kwargs:
# Ensure circular reference is set
kwargs['__ovo'].volume = volume._ovo
else:
kwargs.setdefault('user_id', volume.user_id)
kwargs.setdefault('project_id', volume.project_id)
kwargs['volume_id'] = volume.id
kwargs['volume_size'] = volume.size
kwargs['volume_type_id'] = volume.volume_type_id
kwargs['volume'] = volume._ovo
if 'description' in kwargs:
kwargs['display_description'] = kwargs.pop('description')
if 'name' in kwargs:
kwargs['display_name'] = kwargs.pop('name')
super(Snapshot, self).__init__(volume.backend, **kwargs)
@classmethod
def _load(cls, backend, ovo):
# Turn this around and do a Volume load
volume = ovo.volume
volume.snapshots.objects.append(ovo)
# Remove circular reference
delattr(ovo, base_ovo._get_attrname('volume'))
Volume._load(backend, volume)
return Snapshot.objects[ovo.id]
def _replace_ovo(self, ovo):
super(Snapshot, self)._replace_ovo(ovo)
# Ensure circular reference is set
self._ovo.volume = self.volume._ovo
def create(self):
try:
model_update = self.backend.driver.create_snapshot(self._ovo)
self._ovo.status = 'available'
if model_update:
self._ovo.update(model_update)
except Exception:
self._ovo.status = 'error'
# TODO: raise with the vol info
raise
def delete(self):
try:
self.backend.driver.delete_snapshot(self._ovo)
self._ovo.status = 'deleted'
self._ovo.deleted = True
# snapshot.deleted_at =
except Exception:
self._ovo.status = 'error'
# TODO: raise with the snap info
raise
self.volume.snapshots.discard(self)
try:
self.volume._ovo.snapshots.objects.remove(self._ovo)
except ValueError:
pass
def create_volume(self, **new_vol_params):
new_vol_params['snapshot_id'] = self.id
new_vol = Volume(self.volume, **new_vol_params)
try:
model_update = self.backend.driver.create_volume_from_snapshot(
new_vol._ovo, self._ovo)
new_vol._ovo.status = 'available'
if model_update:
new_vol._ovo.update(model_update)
except Exception:
new_vol._ovo.status = 'error'
# TODO: raise with the new volume info
raise
return new_vol
class OVO(object):
"""Oslo Versioned Objects helper class.
Class will prevent OVOs from actually trying to save to the DB on request,
replace some 'get_by_id' methods to prevent them from going to the DB while
still returned the expected data, remove circular references when saving
objects (for example in a Volume OVO it has a 'snapshot' field which is a
Snapshot OVO that has a 'volume' back reference), piggy back on the OVO's
serialization mechanism to add/get additional data we want.
"""
OBJ_NAME_MAPS = {'VolumeAttachment': 'Connection'}
@classmethod
def _ovo_init(cls, non_uuid_ids):
# Create fake DB for drivers
cls.fake_db = DB()
# Replace the standard DB configuration for code that doesn't use the
# driver.db attribute (ie: OVOs).
volume_cmd.session.IMPL = cls.fake_db
# Replace get_by_id methods with something that will return expected
# data
volume_cmd.objects.Volume.get_by_id = DB.volume_get
volume_cmd.objects.Snapshot.get_by_id = DB.snapshot_get
# Use custom dehydration methods that prevent maximum recursion errors
# due to circular references:
# ie: snapshot -> volume -> snapshots -> snapshot
base_ovo.VersionedObject.obj_to_primitive = cls.obj_to_primitive
cinder_base_ovo.CinderObject.obj_from_primitive = classmethod(
cls.obj_from_primitive)
fields = base_ovo.obj_fields
fields.Object.to_primitive = staticmethod(cls.field_ovo_to_primitive)
fields.Field.to_primitive = cls.field_to_primitive
fields.List.to_primitive = cls.iterable_to_primitive
fields.Set.to_primitive = cls.iterable_to_primitive
fields.Dict.to_primitive = cls.dict_to_primitive
cls.wrap_to_primitive(fields.FieldType)
cls.wrap_to_primitive(fields.DateTime)
cls.wrap_to_primitive(fields.IPAddress)
# Disable saving in ovos
for ovo_name in cinder_base_ovo.CinderObjectRegistry.obj_classes():
ovo_cls = getattr(volume_cmd.objects, ovo_name)
ovo_cls.save = lambda *args, **kwargs: None
if non_uuid_ids and 'id' in ovo_cls.fields:
ovo_cls.fields['id'] = cinder_base_ovo.fields.StringField()
@staticmethod
def wrap_to_primitive(cls):
method = getattr(cls, 'to_primitive')
@functools.wraps(method)
def to_primitive(obj, attr, value, visited=None):
return method(obj, attr, value)
setattr(cls, 'to_primitive', staticmethod(to_primitive))
@staticmethod
def obj_to_primitive(self, target_version=None, version_manifest=None,
visited=None):
# No target_version, version_manifest, or changes support
if visited is None:
visited = set()
visited.add(id(self))
primitive = {}
for name, field in self.fields.items():
if self.obj_attr_is_set(name):
value = getattr(self, name)
# Skip cycles
if id(value) in visited:
continue
primitive[name] = field.to_primitive(self, name, value,
visited)
obj_name = self.obj_name()
obj = {
self._obj_primitive_key('name'): obj_name,
self._obj_primitive_key('namespace'): self.OBJ_PROJECT_NAMESPACE,
self._obj_primitive_key('version'): self.VERSION,
self._obj_primitive_key('data'): primitive
}
# Piggyback to store our own data
my_obj_name = OVO.OBJ_NAME_MAPS.get(obj_name, obj_name)
if 'id' in primitive and my_obj_name in Object.objects:
my_obj = Object.objects[my_obj_name][primitive['id']]
obj['cinderlib.data'] = my_obj._to_primitive()
return obj
@staticmethod
def obj_from_primitive(
cls, primitive, context=None,
original_method=cinder_base_ovo.CinderObject.obj_from_primitive):
result = original_method(primitive, context)
result.cinderlib_data = primitive.get('cinderlib.data')
return result
@staticmethod
def field_ovo_to_primitive(obj, attr, value, visited=None):
return value.obj_to_primitive(visited=visited)
@staticmethod
def field_to_primitive(self, obj, attr, value, visited=None):
if value is None:
return None
return self._type.to_primitive(obj, attr, value, visited)
@staticmethod
def iterable_to_primitive(self, obj, attr, value, visited=None):
if visited is None:
visited = set()
visited.add(id(value))
result = []
for elem in value:
if id(elem) in visited:
continue
visited.add(id(elem))
r = self._element_type.to_primitive(obj, attr, elem, visited)
result.append(r)
return result
@staticmethod
def dict_to_primitive(self, obj, attr, value, visited=None):
if visited is None:
visited = set()
visited.add(id(value))
primitive = {}
for key, elem in value.items():
if id(elem) in visited:
continue
visited.add(id(elem))
primitive[key] = self._element_type.to_primitive(
obj, '%s["%s"]' % (attr, key), elem, visited)
return primitive
class DB(object):
"""Replacement for DB access methods.
This will serve as replacement for methods used by:
- Drivers
- OVOs' get_by_id method
- DB implementation
Data will be retrieved based on the objects we store in our own Volume
and Snapshots classes.
"""
@classmethod
def volume_get(cls, context, volume_id, *args, **kwargs):
if volume_id not in Volume.objects:
raise exception.VolumeNotFound(volume_id=volume_id)
return Volume.objects[volume_id]._ovo
@classmethod
def snapshot_get(cls, context, snapshot_id, *args, **kwargs):
if snapshot_id not in Snapshot.objects:
raise exception.SnapshotNotFound(snapshot_id=snapshot_id)
return Snapshot.objects[snapshot_id]._ovo
@classmethod
def image_volume_cache_get_by_volume_id(cls, context, volume_id):
return None
setup = Backend.global_setup
class MyDict(dict):

29
cinderlib/exception.py Normal file
View File

@@ -0,0 +1,29 @@
# Copyright (c) 2018, Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cinder import exception
NotFound = exception.NotFound
VolumeNotFound = exception.VolumeNotFound
SnapshotNotFound = exception.SnapshotNotFound
ConnectionNotFound = exception.VolumeAttachmentNotFound
class InvalidPersistence(Exception):
__msg = 'Invalid persistence storage: %s'
def __init__(self, name):
super(InvalidPersistence, self).__init__(self.__msg % name)

645
cinderlib/objects.py Normal file
View File

@@ -0,0 +1,645 @@
# Copyright (c) 2018, Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import json as json_lib
import uuid
from cinder import context
# NOTE(geguileo): Probably a good idea not to depend on cinder.cmd.volume
# having all the other imports as they could change.
from cinder.cmd import volume as volume_cmd
from cinder.objects import base as cinder_base_ovo
from cinder import utils
from oslo_versionedobjects import base as base_ovo
from os_brick import exception as brick_exception
from cinderlib import exception
DEFAULT_PROJECT_ID = 'cinderlib'
DEFAULT_USER_ID = 'cinderlib'
# This cannot go in the setup method because cinderlib objects need them to
# be setup to set OVO_CLASS
volume_cmd.objects.register_all()
class Object(object):
"""Base class for our resource representation objects."""
DEFAULT_FIELDS_VALUES = {}
backend_class = None
CONTEXT = context.RequestContext(user_id=DEFAULT_USER_ID,
project_id=DEFAULT_PROJECT_ID,
is_admin=True,
overwrite=False)
def __init__(self, backend, **fields_data):
self.backend = backend
__ovo = fields_data.get('__ovo')
if __ovo:
self._ovo = __ovo
else:
self._ovo = self._create_ovo(**fields_data)
# Store a reference to the cinderlib obj in the OVO for serialization
self._ovo._cl_obj_ = self
@classmethod
def setup(cls, persistence_driver, backend_class, project_id, user_id,
non_uuid_ids):
cls.persistence = persistence_driver
cls.backend_class = backend_class
# Set the global context if we aren't using the default
project_id = project_id or DEFAULT_PROJECT_ID
user_id = user_id or DEFAULT_USER_ID
if (project_id != cls.CONTEXT.project_id or
user_id != cls.CONTEXT.user_id):
cls.CONTEXT.user_id = user_id
cls.CONTEXT.project_id = project_id
Volume.DEFAULT_FIELDS_VALUES['user_id'] = user_id
Volume.DEFAULT_FIELDS_VALUES['project_id'] = project_id
# Configure OVOs to support non_uuid_ids
if non_uuid_ids:
for ovo_name in cinder_base_ovo.CinderObjectRegistry.obj_classes():
ovo_cls = getattr(volume_cmd.objects, ovo_name)
if 'id' in ovo_cls.fields:
ovo_cls.fields['id'] = cinder_base_ovo.fields.StringField()
def _to_primitive(self):
return None
def _create_ovo(self, **fields_data):
# The base are the default values we define on our own classes
fields_values = self.DEFAULT_FIELDS_VALUES.copy()
# Apply the values defined by the caller
fields_values.update(fields_data)
# We support manually setting the id, so set only if not already set
# or if set to None
if not fields_values.get('id'):
fields_values['id'] = self.new_uuid()
# Set non set field values based on OVO's default value and on whether
# it is nullable or not.
for field_name, field in self.OVO_CLASS.fields.items():
if field.default != cinder_base_ovo.fields.UnspecifiedDefault:
fields_values.setdefault(field_name, field.default)
elif field.nullable:
fields_values.setdefault(field_name, None)
return self.OVO_CLASS(context=self.CONTEXT, **fields_values)
@property
def json(self):
ovo = self._ovo.obj_to_primitive()
return {'class': type(self).__name__,
'backend': self.backend.config,
'ovo': ovo}
@property
def jsons(self):
return json_lib.dumps(self.json)
def __repr__(self):
return ('<cinderlib.%s object %s on backend %s>' %
(type(self).__name__,
self.id,
self.backend.id))
@classmethod
def load(cls, json_src):
backend = cls.backend_class.load_backend(json_src['backend'])
backend_name = json_src['backend']['volume_backend_name']
if backend_name in cls.backend_class.backends:
backend = cls.backend_class.backends[backend_name]
elif len(json_src['backend']) == 1:
raise Exception('Backend not present in system or json.')
else:
backend = cls.backend_class(**json_src['backend'])
ovo = cinder_base_ovo.CinderObject.obj_from_primitive(json_src['ovo'],
cls.CONTEXT)
return cls._load(backend, ovo)
def _replace_ovo(self, ovo):
self._ovo = ovo
@staticmethod
def new_uuid():
return str(uuid.uuid4())
def __getattr__(self, name):
if name == '_ovo':
raise AttributeError('Attribute _ovo is not yet set')
return getattr(self._ovo, name)
class Volume(Object):
OVO_CLASS = volume_cmd.objects.Volume
DEFAULT_FIELDS_VALUES = {
'size': 1,
'user_id': Object.CONTEXT.user_id,
'project_id': Object.CONTEXT.project_id,
'host': volume_cmd.CONF.host,
'status': 'creating',
'attach_status': 'detached',
'metadata': {},
'admin_metadata': {},
'glance_metadata': {},
}
_ignore_keys = ('id', 'volume_attachment', 'snapshots')
def __init__(self, backend_or_vol, **kwargs):
# Accept a volume as additional source data
if isinstance(backend_or_vol, Volume):
# Availability zone (backend) will be the same as the source
kwargs.pop('availability_zone', None)
for key in backend_or_vol._ovo.fields:
if (backend_or_vol._ovo.obj_attr_is_set(key) and
key not in self._ignore_keys):
kwargs.setdefault(key, getattr(backend_or_vol._ovo, key))
backend_or_vol = backend_or_vol.backend
if '__ovo' not in kwargs:
if 'description' in kwargs:
kwargs['display_description'] = kwargs.pop('description')
if 'name' in kwargs:
kwargs['display_name'] = kwargs.pop('name')
kwargs.setdefault(
'volume_attachment',
volume_cmd.objects.VolumeAttachmentList(context=self.CONTEXT))
kwargs.setdefault(
'snapshots',
volume_cmd.objects.SnapshotList(context=self.CONTEXT))
super(Volume, self).__init__(backend_or_vol, **kwargs)
self.snapshots = set()
self.connections = []
self._populate_data()
def _to_primitive(self):
local_attach = self.local_attach.id if self.local_attach else None
return {'local_attach': local_attach,
'exported': self.exported}
@classmethod
def get_by_id(cls, volume_id):
result = cls.persistence.get_volumes(volume_id=volume_id)
if not result:
raise exception.VolumeNotFound(volume_id=volume_id)
return result[0]
@classmethod
def get_by_name(cls, volume_name):
return cls.persistence.get_volumes(volume_name=volume_name)
@classmethod
def _load(cls, backend, ovo):
# Restore snapshot's circular reference removed on serialization
for snap in ovo.snapshots:
snap.volume = ovo
# If this object is already present it will be replaced
obj = Object.objects['Volume'].get(ovo.id)
if obj:
obj._replace_ovo(ovo)
else:
obj = cls(backend, __ovo=ovo)
return obj
def _replace_ovo(self, ovo):
super(Volume, self)._replace_ovo(ovo)
self._populate_data()
def _populate_data(self):
old_snapshots = {snap.id: snap for snap in self.snapshots}
for snap_ovo in self._ovo.snapshots:
snap = Object.objects['Snapshot'].get(snap_ovo.id)
if snap:
snap._replace_ovo(snap_ovo)
del old_snapshots[snap.id]
else:
snap = Snapshot(self, __ovo=snap_ovo)
self.snapshots.add(snap)
for snap_id, snap in old_snapshots.items():
self.snapshots.discard(snap)
# We leave snapshots in the global DB just in case...
# del Object.objects['Snapshot'][snap_id]
old_connections = {conn.id: conn for conn in self.connections}
for conn_ovo in self._ovo.volume_attachment:
conn = Object.objects['Connection'].get(conn_ovo.id)
if conn:
conn._replace_ovo(conn_ovo)
del old_connections[conn.id]
else:
conn = Connection(self.backend, volume=self, __ovo=conn_ovo)
self.connections.append(conn)
for conn_id, conn in old_connections.items():
self.connections.remove(conn)
# We leave connections in the global DB just in case...
# del Object.objects['Connection'][conn_id]
data = getattr(self._ovo, 'cinderlib_data', {})
self.exported = data.get('exported', False)
self.local_attach = data.get('local_attach', None)
if self.local_attach:
self.local_attach = Object.objects['Connection'][self.local_attach]
def create(self):
try:
model_update = self.backend.driver.create_volume(self._ovo)
self._ovo.status = 'available'
if model_update:
self._ovo.update(model_update)
except Exception:
self._ovo.status = 'error'
# TODO: raise with the vol info
raise
finally:
self.persistence.set_volume(self)
def delete(self):
# Some backends delete existing snapshots while others leave them
try:
self.backend.driver.delete_volume(self._ovo)
self.persistence.delete_volume(self)
except Exception:
# We don't change status to error on deletion error, we assume it
# just didn't complete.
# TODO: raise with the vol info
raise
def extend(self, size):
volume = self._ovo
volume.previous_status = volume.status
volume.status = 'extending'
try:
self.backend.driver.extend_volume(volume, size)
volume.size = size
volume.status = volume.previous_status
volume.previous_status = None
except Exception:
volume.status = 'error'
# TODO: raise with the vol info
raise
finally:
self.persistence.set_volume(self)
def clone(self, **new_vol_attrs):
new_vol_attrs['source_vol_id'] = self.id
new_vol = Volume(self, **new_vol_attrs)
try:
model_update = self.backend.driver.create_cloned_volume(
new_vol._ovo, self._ovo)
new_vol.status = 'available'
if model_update:
new_vol.update(model_update)
except Exception:
new_vol.status = 'error'
# TODO: raise with the new volume info
raise
finally:
self.persistence.set_volume(new_vol)
return new_vol
def create_snapshot(self, name='', description='', **kwargs):
snap = Snapshot(self, name=name, description=description, **kwargs)
snap.create()
self.snapshots.add(snap)
self._ovo.snapshots.objects.append(snap._ovo)
self.persistence.reset_change_tracker(self, 'snapshots')
return snap
def attach(self):
connector_dict = utils.brick_get_connector_properties(
self.backend.configuration.use_multipath_for_image_xfer,
self.backend.configuration.enforce_multipath_for_image_xfer)
conn = self.connect(connector_dict)
try:
conn.attach()
except Exception:
self.disconnect(conn)
raise
return conn
def detach(self, force=False, ignore_errors=False):
if not self.local_attach:
raise Exception('Not attached')
exc = brick_exception.ExceptionChainer()
conn = self.local_attach
try:
conn.detach(force, ignore_errors, exc)
except Exception:
if not force:
raise
with exc.context(force, 'Unable to disconnect'):
conn.disconnect(force)
if exc and not ignore_errors:
raise exc
def connect(self, connector_dict, **ovo_fields):
if not self.exported:
model_update = self.backend.driver.create_export(self.CONTEXT,
self._ovo,
connector_dict)
if model_update:
self._ovo.update(model_update)
self.persistence.set_volume(self)
self.exported = True
try:
conn = Connection.connect(self, connector_dict, **ovo_fields)
self.connections.append(conn)
self._ovo.status = 'in-use'
self.persistence.set_volume(self)
except Exception:
if not self.connections:
self._remove_export()
# TODO: Improve raised exception
raise
return conn
def _disconnect(self, connection):
self.connections.remove(connection)
if not self.connections:
self._remove_export()
self._ovo.status = 'available'
def disconnect(self, connection, force=False):
connection._disconnect(force)
self._disconnect(connection)
def cleanup(self):
for attach in self.attachments:
attach.detach()
self._remove_export()
def _remove_export(self):
self.backend.driver.remove_export(self._context, self._ovo)
self.exported = False
class Connection(Object):
OVO_CLASS = volume_cmd.objects.VolumeAttachment
@classmethod
def connect(cls, volume, connector, *kwargs):
conn_info = volume.backend.driver.initialize_connection(
volume._ovo, connector)
conn = cls(volume.backend,
connector=connector,
volume=volume,
status='attached',
attach_mode='rw',
connection_info=conn_info,
*kwargs)
cls.persistence.set_connection(conn)
volume._ovo.volume_attachment.objects.append(conn._ovo)
cls.persistence.reset_change_tracker(volume, 'volume_attachment')
return conn
def __init__(self, *args, **kwargs):
self.connected = True
self.volume = kwargs.pop('volume')
self.connector = kwargs.pop('connector', None)
self.attach_info = kwargs.pop('attach_info', None)
if '__ovo' not in kwargs:
kwargs['volume'] = self.volume._ovo
kwargs['volume_id'] = self.volume._ovo.id
super(Connection, self).__init__(*args, **kwargs)
self._populate_data()
def _to_primitive(self):
result = {
'connector': self.connector,
}
if self.attach_info:
attach_info = self.attach_info.copy()
connector = attach_info['connector']
attach_info['connector'] = {
'use_multipath': connector.use_multipath,
'device_scan_attempts': connector.device_scan_attempts,
}
else:
attach_info = None
result['attachment'] = attach_info
return result
def _populate_data(self):
# Ensure circular reference is set
self._ovo.volume = self.volume._ovo
data = getattr(self._ovo, 'cinderlib_data', None)
if data:
self.connector = data.get('connector', None)
self.attach_info = data.get('attachment', None)
conn = (self.attach_info or {}).get('connector')
if isinstance(conn, dict):
self.attach_info['connector'] = utils.brick_get_connector(
self.connection_info['driver_volume_type'],
conn=self.connection_info,
**conn)
self.attached = bool(self.attach_info)
def _replace_ovo(self, ovo):
super(Connection, self)._replace_ovo(ovo)
self._populate_data()
@classmethod
def _load(cls, backend, ovo):
# Turn this around and do a Volume load
volume = ovo.volume
volume.volume_attachment.objects.append(ovo)
# Remove circular reference
delattr(ovo, base_ovo._get_attrname('volume'))
Volume._load(backend, volume)
return Connection.objects[ovo.id]
def _disconnect(self, force=False):
self.backend.driver.terminate_connection(self._ovo.volume,
self.connector,
force=force)
self.connected = False
self._ovo.volume.volume_attachment.objects.remove(self._ovo)
self._ovo.status = 'detached'
self._ovo.deleted = True
self.persistence.delete_connection(self)
def disconnect(self, force=False):
self._disconnect(force)
self.volume._disconnect(self)
def attach(self):
self.attach_info = self.backend.driver._connect_device(
self.connection_info)
self.attached = True
self.volume.local_attach = self
def detach(self, force=False, ignore_errors=False, exc=None):
if not exc:
exc = brick_exception.ExceptionChainer()
connector = self.attach_info['connector']
with exc.context(force, 'Disconnect failed'):
connector.disconnect_volume(self.connection_info['data'],
self.attach_info['device'],
force=force,
ignore_errors=ignore_errors)
self.attached = False
self.volume.local_attach = None
if exc and not ignore_errors:
raise exc
@property
def path(self):
if self.attach_info:
return self.attach_info['device']['path']
return None
@classmethod
def get_by_id(cls, connection_id):
result = cls.persistence.get_connections(connection_id=connection_id)
if not result:
msg = 'id=%s' % connection_id
raise exception.ConnectionNotFound(filter=msg)
return result[0]
class Snapshot(Object):
OVO_CLASS = volume_cmd.objects.Snapshot
DEFAULT_FIELDS_VALUES = {
'status': 'creating',
'metadata': {},
}
def __init__(self, volume, **kwargs):
self.volume = volume
if '__ovo' in kwargs:
# Ensure circular reference is set
kwargs['__ovo'].volume = volume._ovo
else:
kwargs.setdefault('user_id', volume.user_id)
kwargs.setdefault('project_id', volume.project_id)
kwargs['volume_id'] = volume.id
kwargs['volume_size'] = volume.size
kwargs['volume_type_id'] = volume.volume_type_id
kwargs['volume'] = volume._ovo
if 'description' in kwargs:
kwargs['display_description'] = kwargs.pop('description')
if 'name' in kwargs:
kwargs['display_name'] = kwargs.pop('name')
super(Snapshot, self).__init__(volume.backend, **kwargs)
@classmethod
def _load(cls, backend, ovo):
# Turn this around and do a Volume load
volume = ovo.volume
volume.snapshots.objects.append(ovo)
# Remove circular reference
delattr(ovo, base_ovo._get_attrname('volume'))
Volume._load(backend, volume)
return Snapshot.objects[ovo.id]
def _replace_ovo(self, ovo):
super(Snapshot, self)._replace_ovo(ovo)
# Ensure circular reference is set
self._ovo.volume = self.volume._ovo
def create(self):
try:
model_update = self.backend.driver.create_snapshot(self._ovo)
self._ovo.status = 'available'
if model_update:
self._ovo.update(model_update)
except Exception:
self._ovo.status = 'error'
# TODO: raise with the vol info
raise
finally:
self.persistence.set_snapshot(self)
def delete(self):
try:
self.backend.driver.delete_snapshot(self._ovo)
self.persistence.delete_snapshot(self)
except Exception:
# We don't change status to error on deletion error, we assume it
# just didn't complete.
# TODO: raise with the snap info
raise
# Instead of refreshing from the DB we modify the lists manually
self.volume.snapshots.discard(self)
try:
self.volume._ovo.snapshots.objects.remove(self._ovo)
self.persistence.reset_change_tracker(self.volume, 'snapshots')
except ValueError:
pass
def create_volume(self, **new_vol_params):
new_vol_params['snapshot_id'] = self.id
new_vol = Volume(self.volume, **new_vol_params)
try:
model_update = self.backend.driver.create_volume_from_snapshot(
new_vol._ovo, self._ovo)
new_vol._ovo.status = 'available'
if model_update:
new_vol._ovo.update(model_update)
except Exception:
new_vol._ovo.status = 'error'
# TODO: raise with the new volume info
raise
finally:
self.persistence.set_volume(new_vol)
return new_vol
@classmethod
def get_by_id(cls, snapshot_id):
result = cls.persistence.get_snapshots(snapshot_id=snapshot_id)
if not result:
raise exception.SnapshotNotFound(snapshot_id=snapshot_id)
return result[0]
@classmethod
def get_by_name(cls, snapshot_name):
return cls.persistence.get_snapshots(snapshot_name=snapshot_name)
setup = Object.setup
CONTEXT = Object.CONTEXT

66
cinderlib/persistence/__init__.py Normal file
View File

@@ -0,0 +1,66 @@
# Copyright (c) 2018, Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import inspect
import six
from stevedore import driver
from cinderlib import exception
from cinderlib.persistence import base
__all__ = ('setup',)
DEFAULT_STORAGE = 'memory'
def setup(config):
"""Setup persistence to be used in cinderlib.
By default memory persistance will be used, but there are other mechanisms
available and other ways to use custom mechanisms:
- Persistence plugins: Plugin mechanism uses Python entrypoints under
namespace cinderlib.persistence.storage, and cinderlib comes with 3
different mechanisms, "memory", "dbms", and "memory_dbms". To use any of
these one must pass the string name in the storage parameter and any
other configuration as keyword arguments.
- Passing a class that inherits from PersistenceDriverBase as storage
parameter and initialization parameters as keyword arguments.
- Passing an instance that inherits from PersistenceDriverBase as storage
parameter.
"""
if config is None:
config = {}
# Default configuration is using memory storage
storage = config.pop('storage', None) or DEFAULT_STORAGE
if isinstance(storage, base.PersistenceDriverBase):
return storage
if inspect.isclass(storage) and issubclass(storage,
base.PersistenceDriverBase):
return storage(**config)
if not isinstance(storage, six.string_types):
raise exception.InvalidPersistence(storage)
persistence_driver = driver.DriverManager(
namespace='cinderlib.persistence.storage',
name=storage,
invoke_on_load=True,
invoke_kwds=config,
)
return persistence_driver.driver

118
cinderlib/persistence/base.py Normal file
View File

@@ -0,0 +1,118 @@
# Copyright (c) 2018, Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# NOTE(geguileo): Probably a good idea not to depend on cinder.cmd.volume
# having all the other imports as they could change.
from cinder.cmd import volume as volume_cmd
from cinder.objects import base as cinder_base_ovo
from oslo_utils import timeutils
import six
class PersistenceDriverBase(object):
"""Provide Metadata Persistency for our resources.
This class will be used to store new resources as they are created,
updated, and removed, as well as provide a mechanism for users to retrieve
volumes, snapshots, and connections.
"""
@property
def db(self):
raise NotImplemented()
def get_volumes(self, volume_id=None, volume_name=None, backend_name=None):
raise NotImplemented()
def get_snapshots(self, snapshot_id=None, snapshot_name=None,
volume_id=None):
raise NotImplemented()
def get_connections(self, connection_id=None, volume_id=None):
raise NotImplemented()
def set_volume(self, volume):
self.reset_change_tracker(volume)
def set_snapshot(self, snapshot):
self.reset_change_tracker(snapshot)
def set_connection(self, connection):
self.reset_change_tracker(connection)
def delete_volume(self, volume):
self._set_deleted(volume)
self.reset_change_tracker(volume)
def delete_snapshot(self, snapshot):
self._set_deleted(snapshot)
self.reset_change_tracker(snapshot)
def delete_connection(self, connection):
self._set_deleted(connection)
self.reset_change_tracker(connection)
def _set_deleted(self, resource):
resource._ovo.deleted = True
resource._ovo.deleted_at = timeutils.utcnow()
if hasattr(resource._ovo, 'status'):
resource._ovo.status = 'deleted'
def reset_change_tracker(self, resource, fields=None):
if isinstance(fields, six.string_types):
fields = (fields,)
resource._ovo.obj_reset_changes(fields)
def get_changed_fields(self, resource):
return resource._ovo.cinder_obj_get_changes()
class DB(object):
"""Replacement for DB access methods.
This will serve as replacement for methods used by:
- Drivers
- OVOs' get_by_id and save methods
- DB implementation
Data will be retrieved using the persistence driver we setup.
"""
def __init__(self, persistence_driver):
self.persistence = persistence_driver
# Replace the standard DB configuration for code that doesn't use the
# driver.db attribute (ie: OVOs).
volume_cmd.session.IMPL = self
# Replace get_by_id OVO methods with something that will return
# expected data
volume_cmd.objects.Volume.get_by_id = self.volume_get
volume_cmd.objects.Snapshot.get_by_id = self.snapshot_get
# Disable saving in OVOs
for ovo_name in cinder_base_ovo.CinderObjectRegistry.obj_classes():
ovo_cls = getattr(volume_cmd.objects, ovo_name)
ovo_cls.save = lambda *args, **kwargs: None
def volume_get(self, context, volume_id, *args, **kwargs):
return self.persistence.get_volumes(volume_id)._ovo
def snapshot_get(self, context, snapshot_id, *args, **kwargs):
return self.persistence.get_snapshots(snapshot_id)._ovo
@classmethod
def image_volume_cache_get_by_volume_id(cls, context, volume_id):
return None

81
cinderlib/persistence/memory.py Normal file
View File

@@ -0,0 +1,81 @@
# Copyright (c) 2018, Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cinderlib.persistence import base as persistence_base
class MemoryPersistence(persistence_base.PersistenceDriverBase):
volumes = {}
snapshots = {}
connections = {}
def __init__(self):
super(MemoryPersistence, self).__init__()
# Create fake DB for drivers
self.fake_db = persistence_base.DB(self)
@property
def db(self):
return self.fake_db
def _filter_by(self, values, field, value):
if not value:
return values
return [res for res in values if getattr(res, field) == value]
def get_volumes(self, volume_id=None, volume_name=None, backend_name=None):
res = [self.volumes[volume_id]] if volume_id else self.volumes.values()
res = self._filter_by(res, 'display_name', volume_name)
res = self._filter_by(res, 'availability_zone', backend_name)
return res
def get_snapshots(self, snapshot_id=None, snapshot_name=None,
volume_id=None):
result = ([self.snapshots[snapshot_id]] if snapshot_id
else self.snapshots.values())
result = self._filter_by(result, 'volume_id', volume_id)
result = self._filter_by(result, 'display_name', snapshot_name)
return result
def get_connections(self, connection_id=None, volume_id=None):
result = ([self.connections[connection_id]] if connection_id
else self.connections.values())
result = self._filter_by(result, 'volume_id', volume_id)
return result
def set_volume(self, volume):
self.volumes[volume.id] = volume
super(MemoryPersistence, self).set_volume(volume)
def set_snapshot(self, snapshot):
self.snapshots[snapshot.id] = snapshot
super(MemoryPersistence, self).set_snapshot(snapshot)
def set_connection(self, connection):
self.connections[connection.id] = connection
super(MemoryPersistence, self).set_connection(connection)
def delete_volume(self, volume):
self.volumes.pop(volume.id, None)
super(MemoryPersistence, self).delete_volume(volume)
def delete_snapshot(self, snapshot):
self.snapshots.pop(snapshot.id, None)
super(MemoryPersistence, self).delete_snapshot(snapshot)
def delete_connection(self, connection):
self.connections.pop(connection.id, None)
super(MemoryPersistence, self).delete_connection(connection)

169
cinderlib/serialization.py Normal file
View File

@@ -0,0 +1,169 @@
# Copyright (c) 2018, Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Oslo Versioned Objects helper file.
These methods help with the serialization of Cinderlib objects that uses the
OVO serialization mechanism, so we remove circular references when doing the
JSON serialization of objects (for example in a Volume OVO it has a 'snapshot'
field which is a Snapshot OVO that has a 'volume' back reference), piggy back
on the OVO's serialization mechanism to add/get additional data we want.
"""
import functools
import json as json_lib
import six
from cinder.objects import base as cinder_base_ovo
from oslo_versionedobjects import base as base_ovo
from cinderlib import objects
# Variable used to avoid circular references
BACKEND_CLASS = None
def setup(backend_class):
global BACKEND_CLASS
BACKEND_CLASS = backend_class
# Use custom dehydration methods that prevent maximum recursion errors
# due to circular references:
# ie: snapshot -> volume -> snapshots -> snapshot
base_ovo.VersionedObject.obj_to_primitive = obj_to_primitive
cinder_base_ovo.CinderObject.obj_from_primitive = classmethod(
obj_from_primitive)
fields = base_ovo.obj_fields
fields.Object.to_primitive = staticmethod(field_ovo_to_primitive)
fields.Field.to_primitive = field_to_primitive
fields.List.to_primitive = iterable_to_primitive
fields.Set.to_primitive = iterable_to_primitive
fields.Dict.to_primitive = dict_to_primitive
wrap_to_primitive(fields.FieldType)
wrap_to_primitive(fields.DateTime)
wrap_to_primitive(fields.IPAddress)
def wrap_to_primitive(cls):
method = getattr(cls, 'to_primitive')
@functools.wraps(method)
def to_primitive(obj, attr, value, visited=None):
return method(obj, attr, value)
setattr(cls, 'to_primitive', staticmethod(to_primitive))
def obj_to_primitive(self, target_version=None,
version_manifest=None, visited=None):
# No target_version, version_manifest, or changes support
if visited is None:
visited = set()
visited.add(id(self))
primitive = {}
for name, field in self.fields.items():
if self.obj_attr_is_set(name):
value = getattr(self, name)
# Skip cycles
if id(value) in visited:
continue
primitive[name] = field.to_primitive(self, name, value,
visited)
obj_name = self.obj_name()
obj = {
self._obj_primitive_key('name'): obj_name,
self._obj_primitive_key('namespace'): self.OBJ_PROJECT_NAMESPACE,
self._obj_primitive_key('version'): self.VERSION,
self._obj_primitive_key('data'): primitive
}
# Piggyback to store our own data
cl_obj = getattr(self, '_cl_obj', None)
clib_data = cl_obj and cl_obj._to_primitive()
if clib_data:
obj['cinderlib.data'] = clib_data
return obj
def obj_from_primitive(
cls, primitive, context=None,
original_method=cinder_base_ovo.CinderObject.obj_from_primitive):
result = original_method(primitive, context)
result.cinderlib_data = primitive.get('cinderlib.data')
return result
def field_ovo_to_primitive(obj, attr, value, visited=None):
return value.obj_to_primitive(visited=visited)
def field_to_primitive(self, obj, attr, value, visited=None):
if value is None:
return None
return self._type.to_primitive(obj, attr, value, visited)
def iterable_to_primitive(self, obj, attr, value, visited=None):
if visited is None:
visited = set()
visited.add(id(value))
result = []
for elem in value:
if id(elem) in visited:
continue
visited.add(id(elem))
r = self._element_type.to_primitive(obj, attr, elem, visited)
result.append(r)
return result
def dict_to_primitive(self, obj, attr, value, visited=None):
if visited is None:
visited = set()
visited.add(id(value))
primitive = {}
for key, elem in value.items():
if id(elem) in visited:
continue
visited.add(id(elem))
primitive[key] = self._element_type.to_primitive(
obj, '%s["%s"]' % (attr, key), elem, visited)
return primitive
def load(json_src):
"""Load any json serialized cinderlib object."""
if isinstance(json_src, six.string_types):
json_src = json_lib.loads(json_src)
if isinstance(json_src, list):
return [getattr(objects, obj['class']).load(obj) for obj in json_src]
return getattr(objects, json_src['class']).load(json_src)
def json():
"""Convert to Json everything we have in this system."""
return [backend.json for backend in BACKEND_CLASS.backends.values()]
def jsons():
"""Convert to a Json string everything we have in this system."""
return json_lib.dumps(json())

15
setup.py
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from setuptools import setup
import setuptools
with open('README.rst') as readme_file:
readme = readme_file.read()
@@ -47,7 +47,7 @@ extras = {
'infi.dtypes.iqn'],
}
setup(
setuptools.setup(
name='cinderlib',
version='0.1.0',
description=("Cinder Library allows using storage drivers outside of "
@@ -56,11 +56,7 @@ setup(
author="Gorka Eguileor",
author_email='geguileo@redhat.com',
url='https://github.com/akrog/cinderlib',
packages=[
'cinderlib',
],
package_dir={'cinderlib':
'cinderlib'},
packages=setuptools.find_packages(exclude=['tmp', 'tests.*', 'tests.*']),
include_package_data=True,
install_requires=requirements,
extras_requires=extras,
@@ -81,4 +77,9 @@ setup(
],
test_suite='unittest2.collector',
tests_require=test_requirements,
entry_points={
'cinderlib.persistence.storage': [
'memory = cinderlib.persistence.memory:MemoryPersistence',
],
},
)