Add content parameter for checkpoint and bank modules

Add optional content parameter for checkpoint interface, so that other
checkpoint bank plugin like database bank plugin can be introduced to
Karbor.

Change-Id: I4eaad0b3fe38cb95a668b83acba39cd831bbb7df
Closes-Bug: #1745909
This commit is contained in:
chenying 2018-01-31 14:52:10 +08:00
parent 2eeeeafff7
commit d85ec6aa58
19 changed files with 155 additions and 125 deletions

View File

@ -41,24 +41,24 @@ class BankPlugin(object):
self._config = config
@abc.abstractmethod
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
@abc.abstractmethod
def get_object(self, key):
def get_object(self, key, context=None):
return
@abc.abstractmethod
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
return
@abc.abstractmethod
def delete_object(self, key):
def delete_object(self, key, context=None):
return
@abc.abstractmethod
def get_owner_id(self):
def get_owner_id(self, context=None):
return
@ -114,16 +114,18 @@ class Bank(object):
err=_('Invalid parameter: must not contain "." or ".." parts')
)
def update_object(self, key, value):
def update_object(self, key, value, context=None):
self._validate_key(key)
return self._plugin.update_object(self._normalize_key(key), value)
return self._plugin.update_object(self._normalize_key(key), value,
context=context)
def get_object(self, key):
def get_object(self, key, context=None):
self._validate_key(key)
return self._plugin.get_object(self._normalize_key(key))
return self._plugin.get_object(self._normalize_key(key),
context=context)
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
if not prefix:
prefix = "/"
@ -133,12 +135,14 @@ class Bank(object):
prefix=norm_prefix,
limit=limit,
marker=marker,
sort_dir=sort_dir
sort_dir=sort_dir,
context=context
)
def delete_object(self, key):
def delete_object(self, key, context=None):
self._validate_key(key)
return self._plugin.delete_object(self._normalize_key(key))
return self._plugin.delete_object(self._normalize_key(key),
context=context)
def get_sub_section(self, section, is_writable=True):
return BankSection(self, section, is_writable)
@ -229,20 +233,22 @@ class BankSection(object):
if not self.is_writable:
raise exception.BankReadonlyViolation()
def update_object(self, key, value):
def update_object(self, key, value, context=None):
self._validate_writable()
return self._bank.update_object(
self._prepend_prefix(key),
value,
context=context
)
def get_object(self, key):
def get_object(self, key, context=None):
return self._bank.get_object(
self._prepend_prefix(key),
context=context
)
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
if not prefix:
prefix = self._prefix
else:
@ -257,14 +263,16 @@ class BankSection(object):
prefix,
limit,
marker,
sort_dir
sort_dir,
context=context
)
]
def delete_object(self, key):
def delete_object(self, key, context=None):
self._validate_writable()
return self._bank.delete_object(
self._prepend_prefix(key),
context=context
)
def get_owner_id(self):

View File

@ -135,10 +135,10 @@ class FileSystemBankPlugin(BankPlugin):
LOG.exception(_("List the object failed. path: %s"), obj_file_path)
raise
def get_owner_id(self):
def get_owner_id(self, context=None):
return self.owner_id
def update_object(self, key, value):
def update_object(self, key, value, context=None):
LOG.debug("FsBank: update_object. key: %s", key)
self._validate_path(key)
try:
@ -151,7 +151,7 @@ class FileSystemBankPlugin(BankPlugin):
raise exception.BankUpdateObjectFailed(reason=err,
key=key)
def delete_object(self, key):
def delete_object(self, key, context=None):
LOG.debug("FsBank: delete_object. key: %s", key)
self._validate_path(key)
try:
@ -161,7 +161,7 @@ class FileSystemBankPlugin(BankPlugin):
raise exception.BankDeleteObjectFailed(reason=err,
key=key)
def get_object(self, key):
def get_object(self, key, context=None):
LOG.debug("FsBank: get_object. key: %s", key)
self._validate_path(key)
try:
@ -178,7 +178,7 @@ class FileSystemBankPlugin(BankPlugin):
return data
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
LOG.debug("FsBank: list_objects. key: %s", prefix)
try:
file_lists = self._list_object(prefix)

View File

@ -111,10 +111,10 @@ class S3BankPlugin(BankPlugin, LeasePlugin):
)
return self._connection
def get_owner_id(self):
def get_owner_id(self, context=None):
return self.owner_id
def update_object(self, key, value):
def update_object(self, key, value, context=None):
serialized = False
try:
if not isinstance(value, str):
@ -130,7 +130,7 @@ class S3BankPlugin(BankPlugin, LeasePlugin):
LOG.error("update object failed, err: %s.", err)
raise exception.BankUpdateObjectFailed(reason=err, key=key)
def delete_object(self, key):
def delete_object(self, key, context=None):
try:
self._delete_object(bucket=self.bank_object_bucket,
obj=key)
@ -138,7 +138,7 @@ class S3BankPlugin(BankPlugin, LeasePlugin):
LOG.error("delete object failed, err: %s.", err)
raise exception.BankDeleteObjectFailed(reason=err, key=key)
def get_object(self, key):
def get_object(self, key, context=None):
try:
return self._get_object(bucket=self.bank_object_bucket,
obj=key)
@ -147,7 +147,7 @@ class S3BankPlugin(BankPlugin, LeasePlugin):
raise exception.BankGetObjectFailed(reason=err, key=key)
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
try:
response = self._get_bucket(
bucket=self.bank_object_bucket,

View File

@ -108,10 +108,10 @@ class SwiftBankPlugin(BankPlugin, LeasePlugin):
initial_delay=self.lease_renew_window)
return self._connection
def get_owner_id(self):
def get_owner_id(self, context=None):
return self.owner_id
def update_object(self, key, value):
def update_object(self, key, value, context=None):
serialized = False
try:
if not isinstance(value, str):
@ -127,7 +127,7 @@ class SwiftBankPlugin(BankPlugin, LeasePlugin):
LOG.error("update object failed, err: %s.", err)
raise exception.BankUpdateObjectFailed(reason=err, key=key)
def delete_object(self, key):
def delete_object(self, key, context=None):
try:
self._delete_object(container=self.bank_object_container,
obj=key)
@ -135,7 +135,7 @@ class SwiftBankPlugin(BankPlugin, LeasePlugin):
LOG.error("delete object failed, err: %s.", err)
raise exception.BankDeleteObjectFailed(reason=err, key=key)
def get_object(self, key):
def get_object(self, key, context=None):
try:
return self._get_object(container=self.bank_object_container,
obj=key)
@ -144,7 +144,7 @@ class SwiftBankPlugin(BankPlugin, LeasePlugin):
raise exception.BankGetObjectFailed(reason=err, key=key)
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
try:
if sort_dir == "desc":
body = self._get_container(

View File

@ -141,7 +141,7 @@ class Checkpoint(object):
@classmethod
def get_by_section(cls, checkpoints_section, indices_section,
bank_lease, checkpoint_id):
bank_lease, checkpoint_id, context=None):
# TODO(yuvalbr) add validation that the checkpoint exists
checkpoint_section = checkpoints_section.get_sub_section(checkpoint_id)
return Checkpoint(checkpoint_section, indices_section,
@ -168,7 +168,8 @@ class Checkpoint(object):
@classmethod
def create_in_section(cls, checkpoints_section, indices_section,
bank_lease, owner_id, plan,
checkpoint_id=None, checkpoint_properties=None):
checkpoint_id=None, checkpoint_properties=None,
context=None):
checkpoint_id = checkpoint_id or cls._generate_id()
checkpoint_section = checkpoints_section.get_sub_section(checkpoint_id)
@ -202,39 +203,44 @@ class Checkpoint(object):
"extra_info": extra_info,
"created_at": created_at,
"timestamp": timestamp
}
},
context=context
)
indices_section.update_object(
key=cls._get_checkpoint_path_by_provider(
provider_id, project_id, timestamp, checkpoint_id),
value=checkpoint_id
value=checkpoint_id,
context=context
)
indices_section.update_object(
key=cls._get_checkpoint_path_by_date(
created_at, project_id, timestamp, checkpoint_id),
value=checkpoint_id
value=checkpoint_id,
context=context
)
indices_section.update_object(
key=cls._get_checkpoint_path_by_plan(
plan.get("id"), project_id, created_at, timestamp,
checkpoint_id),
value=checkpoint_id)
value=checkpoint_id,
context=context)
return Checkpoint(checkpoint_section,
indices_section,
bank_lease,
checkpoint_id)
def commit(self):
def commit(self, context=None):
self._checkpoint_section.update_object(
key=_INDEX_FILE_NAME,
value=self._md_cache,
context=context
)
def purge(self):
def purge(self, context=None):
"""Purge the index file of the checkpoint.
Can only be done if the checkpoint has no other files apart from the
@ -261,9 +267,9 @@ class Checkpoint(object):
else:
raise RuntimeError(_("Could not delete: Checkpoint is not empty"))
def delete(self):
def delete(self, context=None):
self.status = constants.CHECKPOINT_STATUS_DELETED
self.commit()
self.commit(context=context)
# delete indices
created_at = self._md_cache["created_at"]
timestamp = self._md_cache["timestamp"]
@ -272,13 +278,16 @@ class Checkpoint(object):
project_id = self._md_cache["project_id"]
self._indices_section.delete_object(
self._get_checkpoint_path_by_provider(
provider_id, project_id, timestamp, self.id))
provider_id, project_id, timestamp, self.id),
context=context)
self._indices_section.delete_object(
self._get_checkpoint_path_by_date(
created_at, project_id, timestamp, self.id))
created_at, project_id, timestamp, self.id),
context=context)
self._indices_section.delete_object(
self._get_checkpoint_path_by_plan(
plan_id, project_id, created_at, timestamp, self.id))
plan_id, project_id, created_at, timestamp, self.id),
context=context)
def get_resource_bank_section(self, resource_id):
prefix = "/resource-data/%s/" % resource_id
@ -295,7 +304,8 @@ class CheckpointCollection(object):
self._indices_section = bank.get_sub_section("/indices")
def list_ids(self, project_id, provider_id, limit=None, marker=None,
plan_id=None, start_date=None, end_date=None, sort_dir=None):
plan_id=None, start_date=None, end_date=None, sort_dir=None,
context=None):
marker_checkpoint = None
if marker is not None:
checkpoint_section = self._checkpoints_section.get_sub_section(
@ -325,23 +335,25 @@ class CheckpointCollection(object):
marker = "/by-date/%s/%s/%s" % (date, project_id, marker)
return self._list_ids(project_id, prefix, limit, marker, start_date,
end_date, sort_dir)
end_date, sort_dir, context=context)
def _list_ids(self, project_id, prefix, limit, marker, start_date,
end_date, sort_dir):
end_date, sort_dir, context=None):
if start_date is None:
return [key[key.find("@") + 1:]
for key in self._indices_section.list_objects(
prefix=prefix,
limit=limit,
marker=marker,
sort_dir=sort_dir
sort_dir=sort_dir,
context=context
)]
else:
ids = []
for key in self._indices_section.list_objects(prefix=prefix,
marker=marker,
sort_dir=sort_dir):
sort_dir=sort_dir,
context=context):
date = datetime.strptime(key.split("/")[-3], "%Y-%m-%d")
checkpoint_project_id = key.split("/")[-2]
if start_date <= date <= end_date and (
@ -351,14 +363,15 @@ class CheckpointCollection(object):
return ids
return ids
def get(self, checkpoint_id):
def get(self, checkpoint_id, context=None):
# TODO(saggi): handle multiple instances of the same checkpoint
return Checkpoint.get_by_section(self._checkpoints_section,
self._indices_section,
self._bank_lease,
checkpoint_id)
checkpoint_id,
context=context)
def create(self, plan, checkpoint_properties=None):
def create(self, plan, checkpoint_properties=None, context=None):
# TODO(saggi): Serialize plan to checkpoint. Will be done in
# future patches.
return Checkpoint.create_in_section(
@ -367,4 +380,5 @@ class CheckpointCollection(object):
self._bank_lease,
self._bank.get_owner_id(),
plan,
checkpoint_properties=checkpoint_properties)
checkpoint_properties=checkpoint_properties,
context=context)

View File

@ -107,7 +107,8 @@ class ProtectionManager(manager.Manager):
checkpoint_collection = provider.get_checkpoint_collection()
try:
checkpoint = checkpoint_collection.create(plan,
checkpoint_properties)
checkpoint_properties,
context=context)
except Exception as e:
LOG.exception("Failed to create checkpoint, plan: %s", plan_id)
exc = exception.FlowError(flow="protect",
@ -318,7 +319,8 @@ class ProtectionManager(manager.Manager):
provider = self.provider_registry.show_provider(provider_id)
try:
checkpoint_collection = provider.get_checkpoint_collection()
checkpoint = checkpoint_collection.get(checkpoint_id)
checkpoint = checkpoint_collection.get(checkpoint_id,
context=context)
except Exception:
LOG.error("get checkpoint failed, checkpoint_id:%s",
checkpoint_id)
@ -384,10 +386,11 @@ class ProtectionManager(manager.Manager):
checkpoint_ids = provider.list_checkpoints(
project_id, provider_id, limit=limit, marker=marker,
plan_id=plan_id, start_date=start_date, end_date=end_date,
sort_dir=sort_dir)
sort_dir=sort_dir, context=context)
checkpoints = []
for checkpoint_id in checkpoint_ids:
checkpoint = provider.get_checkpoint(checkpoint_id)
checkpoint = provider.get_checkpoint(checkpoint_id,
context=context)
checkpoints.append(checkpoint.to_dict())
return checkpoints
@ -397,7 +400,7 @@ class ProtectionManager(manager.Manager):
def show_checkpoint(self, context, provider_id, checkpoint_id):
provider = self.provider_registry.show_provider(provider_id)
checkpoint = provider.get_checkpoint(checkpoint_id)
checkpoint = provider.get_checkpoint(checkpoint_id, context=context)
checkpoint_dict = checkpoint.to_dict()
if not context.is_admin and (
context.project_id != checkpoint_dict['project_id']):

View File

@ -144,17 +144,18 @@ class PluggableProtectionProvider(object):
def get_checkpoint_collection(self):
return self.checkpoint_collection
def get_checkpoint(self, checkpoint_id):
return self.get_checkpoint_collection().get(checkpoint_id)
def get_checkpoint(self, checkpoint_id, context=None):
return self.get_checkpoint_collection().get(checkpoint_id,
context=context)
def list_checkpoints(self, project_id, provider_id, limit=None,
marker=None, plan_id=None, start_date=None,
end_date=None, sort_dir=None):
end_date=None, sort_dir=None, context=None):
checkpoint_collection = self.get_checkpoint_collection()
return checkpoint_collection.list_ids(
project_id=project_id, provider_id=provider_id, limit=limit,
marker=marker, plan_id=plan_id, start_date=start_date,
end_date=end_date, sort_dir=sort_dir)
end_date=end_date, sort_dir=sort_dir, context=context)
class ProviderRegistry(object):

View File

@ -25,18 +25,18 @@ class FakeBankPlugin(bank_plugin.BankPlugin):
config.register_opts(fake_bank_opts, 'fake_bank')
self.fake_host = config['fake_bank']['fake_host']
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
def get_object(self, key):
def get_object(self, key, context=None):
return
def list_objects(self, prefix=None, limit=None,
marker=None, sort_dir=None):
marker=None, sort_dir=None, context=None):
return
def delete_object(self, key):
def delete_object(self, key, context=None):
return
def get_owner_id(self):
def get_owner_id(self, context=None):
return

View File

@ -86,16 +86,17 @@ class FakeBankPlugin(BankPlugin):
config.register_opts(fake_bank_opts, 'fake_bank')
self.fake_host = config['fake_bank']['fake_host']
def update_object(self, key, value):
def update_object(self, key, value, context=None):
self._objects[key] = value
def get_object(self, key):
def get_object(self, key, context=None):
value = self._objects.get(key, None)
if value is None:
raise Exception
return value
def list_objects(self, prefix=None, limit=None, marker=None):
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None, context=None):
objects_name = []
if prefix is not None:
for key, value in self._objects.items():
@ -105,10 +106,10 @@ class FakeBankPlugin(BankPlugin):
objects_name = self._objects.keys()
return objects_name
def delete_object(self, key):
def delete_object(self, key, context=None):
self._objects.pop(key)
def get_owner_id(self):
def get_owner_id(self, context=None):
return
@ -276,10 +277,12 @@ class FakeCheckpoint(object):
class FakeCheckpointCollection(object):
def create(self, plan, checkpoint_properties=None):
def create(self, plan, checkpoint_properties=None,
context=None):
return FakeCheckpoint()
def get(self, checkpoint_id):
def get(self, checkpoint_id,
context=None):
return FakeCheckpoint()

View File

@ -27,17 +27,17 @@ class _InMemoryBankPlugin(BankPlugin):
super(_InMemoryBankPlugin, self).__init__(config)
self._data = OrderedDict()
def update_object(self, key, value):
def update_object(self, key, value, context=None):
self._data[key] = value
def get_object(self, key):
def get_object(self, key, context=None):
try:
return deepcopy(self._data[key])
except KeyError:
raise exception.BankGetObjectFailed('no such object')
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
marker_found = marker is None
for key in self._data.keys():
if marker is not True and key != marker:
@ -51,7 +51,7 @@ class _InMemoryBankPlugin(BankPlugin):
else:
marker_found = True
def delete_object(self, key):
def delete_object(self, key, context=None):
del self._data[key]
def get_owner_id(self):

View File

@ -31,20 +31,20 @@ from oslo_config import fixture
class FakeBankPlugin(BankPlugin):
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
def get_object(self, key):
def get_object(self, key, context=None):
return
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
return
def delete_object(self, key):
def delete_object(self, key, context=None):
return
def get_owner_id(self):
def get_owner_id(self, context=None):
return

View File

@ -29,20 +29,20 @@ from oslo_config import fixture
class FakeBankPlugin(BankPlugin):
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
def get_object(self, key):
def get_object(self, key, context=None):
return
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
return
def delete_object(self, key):
def delete_object(self, key, context=None):
return
def get_owner_id(self):
def get_owner_id(self, context=None):
return

View File

@ -29,20 +29,20 @@ from oslo_config import fixture
class FakeBankPlugin(BankPlugin):
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
def get_object(self, key):
def get_object(self, key, context=None):
return
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
return
def delete_object(self, key):
def delete_object(self, key, context=None):
return
def get_owner_id(self):
def get_owner_id(self, context=None):
return

View File

@ -31,20 +31,20 @@ from oslo_config import fixture
class FakeBankPlugin(BankPlugin):
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
def get_object(self, key):
def get_object(self, key, context=None):
return
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
return
def delete_object(self, key):
def delete_object(self, key, context=None):
return
def get_owner_id(self):
def get_owner_id(self, context=None):
return

View File

@ -29,20 +29,20 @@ from oslo_config import fixture
class FakeBankPlugin(BankPlugin):
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
def get_object(self, key):
def get_object(self, key, context=None):
return
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
return
def delete_object(self, key):
def delete_object(self, key, context=None):
return
def get_owner_id(self):
def get_owner_id(self, context=None):
return

View File

@ -29,20 +29,20 @@ from oslo_config import fixture
class FakeBankPlugin(BankPlugin):
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
def get_object(self, key):
def get_object(self, key, context=None):
return
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
return
def delete_object(self, key):
def delete_object(self, key, context=None):
return
def get_owner_id(self):
def get_owner_id(self, context=None):
return

View File

@ -227,16 +227,17 @@ class FakeBankPlugin(BankPlugin):
def create_object(self, key, value):
self._objects[key] = value
def update_object(self, key, value):
def update_object(self, key, value, context=None):
self._objects[key] = value
def get_object(self, key):
def get_object(self, key, context=None):
value = self._objects.get(key, None)
if value is None:
raise Exception
return value
def list_objects(self, prefix=None, limit=None, marker=None):
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None, context=None):
objects_name = []
if prefix is not None:
for key, value in self._objects.items():
@ -246,10 +247,10 @@ class FakeBankPlugin(BankPlugin):
objects_name = self._objects.keys()
return objects_name
def delete_object(self, key):
def delete_object(self, key, context=None):
self._objects.pop(key)
def get_owner_id(self):
def get_owner_id(self, context=None):
return
fake_checkpointid = "checkpoint_id"

View File

@ -209,17 +209,17 @@ class FakeBankPlugin(BankPlugin):
super(FakeBankPlugin, self).__init__(config)
self._objects = {}
def update_object(self, key, value):
def update_object(self, key, value, context=None):
self._objects[key] = value
def get_object(self, key):
def get_object(self, key, context=None):
value = self._objects.get(key, None)
if value is None:
raise Exception
return value
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
objects_name = []
if prefix is not None:
for key, value in self._objects.items():
@ -229,10 +229,10 @@ class FakeBankPlugin(BankPlugin):
objects_name = self._objects.keys()
return objects_name
def delete_object(self, key):
def delete_object(self, key, context=None):
self._objects.pop(key)
def get_owner_id(self):
def get_owner_id(self, context=None):
return

View File

@ -36,20 +36,20 @@ from oslo_config import fixture
class FakeBankPlugin(BankPlugin):
def update_object(self, key, value):
def update_object(self, key, value, context=None):
return
def get_object(self, key):
def get_object(self, key, context=None):
return
def list_objects(self, prefix=None, limit=None, marker=None,
sort_dir=None):
sort_dir=None, context=None):
return
def delete_object(self, key):
def delete_object(self, key, context=None):
return
def get_owner_id(self):
def get_owner_id(self, context=None):
return