Application Data Protection as a Service in OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

checkpoint.py 14KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386
  1. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  2. # not use this file except in compliance with the License. You may obtain
  3. # a copy of the License at
  4. #
  5. # http://www.apache.org/licenses/LICENSE-2.0
  6. #
  7. # Unless required by applicable law or agreed to in writing, software
  8. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  9. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  10. # License for the specific language governing permissions and limitations
  11. # under the License.
  12. from datetime import datetime
  13. from karbor.common import constants
  14. from karbor import exception
  15. from karbor.i18n import _
  16. from karbor.services.protection import graph
  17. from oslo_config import cfg
  18. from oslo_log import log as logging
  19. from oslo_utils import timeutils
  20. from oslo_utils import uuidutils
  21. CONF = cfg.CONF
  22. LOG = logging.getLogger(__name__)
  23. _INDEX_FILE_NAME = "index.json"
  24. _UUID_STR_LEN = 36
  25. class Checkpoint(object):
  26. VERSION = "0.9"
  27. SUPPORTED_VERSIONS = ["0.9"]
  28. def __init__(self, checkpoint_section, indices_section,
  29. bank_lease, checkpoint_id):
  30. super(Checkpoint, self).__init__()
  31. self._id = checkpoint_id
  32. self._checkpoint_section = checkpoint_section
  33. self._indices_section = indices_section
  34. self._bank_lease = bank_lease
  35. self.reload_meta_data()
  36. def to_dict(self):
  37. return {
  38. "id": self.id,
  39. "status": self.status,
  40. "protection_plan": self.protection_plan,
  41. "extra_info": self._md_cache.get("extra_info", None),
  42. "project_id": self.project_id,
  43. "resource_graph": self._md_cache.get("resource_graph", None),
  44. "created_at": self._md_cache.get("created_at", None)
  45. }
  46. @property
  47. def checkpoint_section(self):
  48. return self._checkpoint_section
  49. @property
  50. def id(self):
  51. return self._id
  52. @property
  53. def provider_id(self):
  54. return self._md_cache["provider_id"]
  55. @property
  56. def created_at(self):
  57. return self._md_cache["created_at"]
  58. @property
  59. def status(self):
  60. # TODO(saggi): check for valid values and transitions
  61. return self._md_cache["status"]
  62. @property
  63. def extra_info(self):
  64. return self._md_cache["extra_info"]
  65. @property
  66. def project_id(self):
  67. return self._md_cache["project_id"]
  68. @property
  69. def owner_id(self):
  70. # TODO(yinwei): check for valid values and transitions
  71. return self._md_cache["owner_id"]
  72. @property
  73. def resource_graph(self):
  74. serialized_resource_graph = self._md_cache.get("resource_graph", None)
  75. if serialized_resource_graph is not None:
  76. resource_graph = graph.deserialize_resource_graph(
  77. serialized_resource_graph)
  78. return resource_graph
  79. else:
  80. return None
  81. @property
  82. def protection_plan(self):
  83. return self._md_cache["protection_plan"]
  84. @status.setter
  85. def status(self, value):
  86. self._md_cache["status"] = value
  87. @extra_info.setter
  88. def extra_info(self, value):
  89. self._md_cache["extra_info"] = value
  90. @resource_graph.setter
  91. def resource_graph(self, resource_graph):
  92. serialized_resource_graph = graph.serialize_resource_graph(
  93. resource_graph)
  94. self._md_cache["resource_graph"] = serialized_resource_graph
  95. def _is_supported_version(self, version):
  96. return version in self.SUPPORTED_VERSIONS
  97. def _assert_supported_version(self, new_md):
  98. if new_md["version"] not in self.SUPPORTED_VERSIONS:
  99. # Something bad happened invalidate the object
  100. self._md_cache = None
  101. self._checkpoint_section = None
  102. raise RuntimeError(
  103. _("Checkpoint was created in an unsupported version"))
  104. def reload_meta_data(self):
  105. try:
  106. new_md = self._checkpoint_section.get_object(_INDEX_FILE_NAME)
  107. except exception.BankGetObjectFailed:
  108. LOG.error("unable to reload metadata for checkpoint id: %s",
  109. self.id)
  110. raise exception.CheckpointNotFound(checkpoint_id=self.id)
  111. self._assert_supported_version(new_md)
  112. self._md_cache = new_md
  113. @classmethod
  114. def _generate_id(self):
  115. return uuidutils.generate_uuid()
  116. @classmethod
  117. def get_by_section(cls, checkpoints_section, indices_section,
  118. bank_lease, checkpoint_id, context=None):
  119. # TODO(yuvalbr) add validation that the checkpoint exists
  120. checkpoint_section = checkpoints_section.get_sub_section(checkpoint_id)
  121. return Checkpoint(checkpoint_section, indices_section,
  122. bank_lease, checkpoint_id)
  123. @staticmethod
  124. def _get_checkpoint_path_by_provider(
  125. provider_id, project_id, timestamp, checkpoint_id):
  126. return "/by-provider/%s/%s/%s@%s" % (
  127. provider_id, project_id, timestamp, checkpoint_id)
  128. @staticmethod
  129. def _get_checkpoint_path_by_plan(
  130. plan_id, project_id, created_at, timestamp, checkpoint_id):
  131. return "/by-plan/%s/%s/%s/%s@%s" % (
  132. plan_id, project_id, created_at, timestamp, checkpoint_id)
  133. @staticmethod
  134. def _get_checkpoint_path_by_date(
  135. created_at, project_id, timestamp, checkpoint_id):
  136. return "/by-date/%s/%s/%s@%s" % (
  137. created_at, project_id, timestamp, checkpoint_id)
  138. @classmethod
  139. def create_in_section(cls, checkpoints_section, indices_section,
  140. bank_lease, owner_id, plan,
  141. checkpoint_id=None, checkpoint_properties=None,
  142. context=None):
  143. checkpoint_id = checkpoint_id or cls._generate_id()
  144. checkpoint_section = checkpoints_section.get_sub_section(checkpoint_id)
  145. timestamp = timeutils.utcnow_ts()
  146. created_at = timeutils.utcnow().strftime('%Y-%m-%d')
  147. provider_id = plan.get("provider_id")
  148. project_id = plan.get("project_id")
  149. extra_info = None
  150. checkpoint_status = constants.CHECKPOINT_STATUS_PROTECTING
  151. if checkpoint_properties:
  152. extra_info = checkpoint_properties.get("extra_info", None)
  153. status = checkpoint_properties.get("status", None)
  154. if status:
  155. checkpoint_status = status
  156. checkpoint_section.update_object(
  157. key=_INDEX_FILE_NAME,
  158. value={
  159. "version": cls.VERSION,
  160. "id": checkpoint_id,
  161. "status": checkpoint_status,
  162. "owner_id": owner_id,
  163. "provider_id": provider_id,
  164. "project_id": project_id,
  165. "protection_plan": {
  166. "id": plan.get("id"),
  167. "name": plan.get("name"),
  168. "provider_id": plan.get("provider_id"),
  169. "resources": plan.get("resources")
  170. },
  171. "extra_info": extra_info,
  172. "created_at": created_at,
  173. "timestamp": timestamp
  174. },
  175. context=context
  176. )
  177. indices_section.update_object(
  178. key=cls._get_checkpoint_path_by_provider(
  179. provider_id, project_id, timestamp, checkpoint_id),
  180. value=checkpoint_id,
  181. context=context
  182. )
  183. indices_section.update_object(
  184. key=cls._get_checkpoint_path_by_date(
  185. created_at, project_id, timestamp, checkpoint_id),
  186. value=checkpoint_id,
  187. context=context
  188. )
  189. indices_section.update_object(
  190. key=cls._get_checkpoint_path_by_plan(
  191. plan.get("id"), project_id, created_at, timestamp,
  192. checkpoint_id),
  193. value=checkpoint_id,
  194. context=context)
  195. return Checkpoint(checkpoint_section,
  196. indices_section,
  197. bank_lease,
  198. checkpoint_id)
  199. def commit(self, context=None):
  200. self._checkpoint_section.update_object(
  201. key=_INDEX_FILE_NAME,
  202. value=self._md_cache,
  203. context=context
  204. )
  205. def purge(self, context=None):
  206. """Purge the index file of the checkpoint.
  207. Can only be done if the checkpoint has no other files apart from the
  208. index.
  209. """
  210. all_objects = self._checkpoint_section.list_objects()
  211. if len(all_objects) == 1 and all_objects[0] == _INDEX_FILE_NAME:
  212. created_at = self._md_cache["created_at"]
  213. timestamp = self._md_cache["timestamp"]
  214. plan_id = self._md_cache["protection_plan"]["id"]
  215. provider_id = self._md_cache["protection_plan"]["provider_id"]
  216. project_id = self._md_cache["project_id"]
  217. self._indices_section.delete_object(
  218. self._get_checkpoint_path_by_provider(
  219. provider_id, project_id, timestamp, self.id))
  220. self._indices_section.delete_object(
  221. self._get_checkpoint_path_by_date(
  222. created_at, project_id, timestamp, self.id))
  223. self._indices_section.delete_object(
  224. self._get_checkpoint_path_by_plan(
  225. plan_id, project_id, created_at, timestamp, self.id))
  226. self._checkpoint_section.delete_object(_INDEX_FILE_NAME)
  227. else:
  228. raise RuntimeError(_("Could not delete: Checkpoint is not empty"))
  229. def delete(self, context=None):
  230. self.status = constants.CHECKPOINT_STATUS_DELETED
  231. self.commit(context=context)
  232. # delete indices
  233. created_at = self._md_cache["created_at"]
  234. timestamp = self._md_cache["timestamp"]
  235. plan_id = self._md_cache["protection_plan"]["id"]
  236. provider_id = self._md_cache["protection_plan"]["provider_id"]
  237. project_id = self._md_cache["project_id"]
  238. self._indices_section.delete_object(
  239. self._get_checkpoint_path_by_provider(
  240. provider_id, project_id, timestamp, self.id),
  241. context=context)
  242. self._indices_section.delete_object(
  243. self._get_checkpoint_path_by_date(
  244. created_at, project_id, timestamp, self.id),
  245. context=context)
  246. self._indices_section.delete_object(
  247. self._get_checkpoint_path_by_plan(
  248. plan_id, project_id, created_at, timestamp, self.id),
  249. context=context)
  250. def get_resource_bank_section(self, resource_id):
  251. prefix = "/resource-data/%s/" % resource_id
  252. return self._checkpoint_section.get_sub_section(prefix)
  253. class CheckpointCollection(object):
  254. def __init__(self, bank, bank_lease=None):
  255. super(CheckpointCollection, self).__init__()
  256. self._bank = bank
  257. self._bank_lease = bank_lease
  258. self._checkpoints_section = bank.get_sub_section("/checkpoints")
  259. self._indices_section = bank.get_sub_section("/indices")
  260. def list_ids(self, project_id, provider_id, limit=None, marker=None,
  261. plan_id=None, start_date=None, end_date=None, sort_dir=None,
  262. context=None):
  263. marker_checkpoint = None
  264. if marker is not None:
  265. checkpoint_section = self._checkpoints_section.get_sub_section(
  266. marker)
  267. marker_checkpoint = checkpoint_section.get_object(_INDEX_FILE_NAME)
  268. timestamp = marker_checkpoint["timestamp"]
  269. marker = "%s@%s" % (timestamp, marker)
  270. if start_date is not None:
  271. if end_date is None:
  272. end_date = timeutils.utcnow()
  273. if plan_id is None and start_date is None:
  274. prefix = "/by-provider/%s/%s/" % (provider_id, project_id)
  275. if marker is not None:
  276. marker = "/%s" % marker
  277. elif plan_id is not None:
  278. prefix = "/by-plan/%s/%s/" % (plan_id, project_id)
  279. if marker is not None:
  280. date = marker_checkpoint["created_at"]
  281. marker = "/%s/%s" % (date, marker)
  282. else:
  283. prefix = "/by-date/"
  284. if marker is not None:
  285. date = marker_checkpoint["created_at"]
  286. marker = "/%s/%s/%s" % (date, project_id, marker)
  287. return self._list_ids(project_id, prefix, limit, marker, start_date,
  288. end_date, sort_dir, context=context)
  289. def _list_ids(self, project_id, prefix, limit, marker, start_date,
  290. end_date, sort_dir, context=None):
  291. if start_date is None:
  292. return [key[key.find("@") + 1:]
  293. for key in self._indices_section.list_objects(
  294. prefix=prefix,
  295. limit=limit,
  296. marker=marker,
  297. sort_dir=sort_dir,
  298. context=context
  299. )]
  300. else:
  301. ids = []
  302. for key in self._indices_section.list_objects(prefix=prefix,
  303. marker=marker,
  304. sort_dir=sort_dir,
  305. context=context):
  306. date_cursor = -2 if (prefix.find('by-plan') >= 0) else -3
  307. project_id_cursor = -3 if (prefix.find('by-plan') >= 0) else -2
  308. date = datetime.strptime(
  309. key.split("/")[date_cursor], "%Y-%m-%d")
  310. checkpoint_project_id = key.split("/")[project_id_cursor]
  311. if start_date <= date <= end_date and (
  312. checkpoint_project_id == project_id):
  313. ids.append(key[key.find("@") + 1:])
  314. if limit is not None and len(ids) == limit:
  315. return ids
  316. return ids
  317. def get(self, checkpoint_id, context=None):
  318. # TODO(saggi): handle multiple instances of the same checkpoint
  319. return Checkpoint.get_by_section(self._checkpoints_section,
  320. self._indices_section,
  321. self._bank_lease,
  322. checkpoint_id,
  323. context=context)
  324. def create(self, plan, checkpoint_properties=None, context=None):
  325. # TODO(saggi): Serialize plan to checkpoint. Will be done in
  326. # future patches.
  327. return Checkpoint.create_in_section(
  328. self._checkpoints_section,
  329. self._indices_section,
  330. self._bank_lease,
  331. self._bank.get_owner_id(),
  332. plan,
  333. checkpoint_properties=checkpoint_properties,
  334. context=context)