diff --git a/doc/source/api/bank.md b/doc/source/api/bank.md new file mode 100644 index 00000000..479dd860 --- /dev/null +++ b/doc/source/api/bank.md @@ -0,0 +1,139 @@ +# Bank basics + +*** :exclamation: This is still a work in progress *** + +This document will describe the layout and algorithms used by smaug using the +bank interface. + +## Abstract + +Since Smaug want's to be able to store metadata in many locations (swift, mongodb, etc.) +we defined a simplified object store interface that we believe most backends will be able +to support without much work. + +But the simplified interface doesn't describe how Smaug will do it's higher +level operations and how the higher level logic will be layed out in the object +store. This is why we need higher level logic defined explicitly so that later +we could use higher level bank functions knowing they are correct, safe and atomic. + +## Layout + +### Checkpoint directory + +`/checkpoints//index.json` + +#### Example content + +```json +{ + "trigger": {}, + "started_at": "11 OCT 2237", + "status": "in progress", + "plan": {}, + "plugins": { + "plugin_1": { + "name": "cinder volume", + "version": "1.2.3", + } + } +} +``` + +### Protection definition directory + +`/checkpoints///index.json` + +#### Example content + +```json +{ + "plugin": "plugin_1", + "dependencies": { + "vol1": "" + } +} +``` + +### Protection definition plugin data directory + +`/checkpoints///plugin_data/*` + +## Checkpoint Creation Process + +Create new Checkpoint with id ; + +1. Acquire checkpoint lock + * action acquire_lock + * id: `` +2. Create checkpoint pointer + * action: write_object + * path: `/indices/unfinished_checkpoints/`, + * buffer: `` +3. Create checkpoint + * action: write_object + * path: `/checkpoints//index.json`, + * buffer: + ```json + { + "smaug_version": "1.0.0", + "status": "in_progress", + "plugins": {} + } + ``` +4. Run plugins +5. Checkpoint finished but indices not yet created + * action: write_object + * path: `/checkpoints//index.json`, + * buffer: + ```json + { + "smaug_version": "1.0.0", + "status": "creating_indices", + "plugins": {} + } + ``` +6. Create index 'plan' (Example, there could be any number of indexes) + * action: write_object + * path: `/indices/by_plan//` + * buffer: `` +7. Remove checkpoint pointer + * action: delete_object + * path: `/indices/unfinished_checkpoints/` +8. Release checkpoint lock + * action: release_lock + * id: `` + +## Delete Checkpoint + +1. Acquire checkpoint lock + * action acquire_lock + * id: `` +2. Create checkpoint pointer + * action: write_object + * path: `/indices/unfinished_checkpoints/`, + * buffer: `` +3. Mark transaction as being deleted + * action: write_object + * path: `/checkpoints//index.json`, + * buffer: + ```json + { + "smaug_version": "1.0.0", + "status": "deleting", + "plugins": {} + } + ``` +4. Remove indices + - Remove index 'plan' (Example, there could be any number of indexes) + * action: delete_object + * path: `/indices/by_plan//` +6. Run plugins +7. Delete checkpoint file + * action: delete_object + * path: `/checkpoints//index.json`, +8. Remove checkpoints pointer + * action: delete_object + * path: `/indices/unfinished_checkpoints/` +9. Release checkpoint lock + * action: release_lock + * id: `` diff --git a/doc/source/api/class_diagram.pu b/doc/source/api/class_diagram.pu new file mode 100644 index 00000000..b9147cdc --- /dev/null +++ b/doc/source/api/class_diagram.pu @@ -0,0 +1,146 @@ +@startuml + +title "Smaug API model" + +class Protectable { + name: string + instances: []Resource + is_root: bool +} + +Protectable --> Resource: lists + +class Resource { + id: UUID + type: ResourceType + schema: JSONSchema + dependent_resources: []Resource +} + +class Trigger { +} + +class TimedTrigger extends Trigger { + +} + +class EventTrigger extends Trigger { + +} + +class Checkpoint { + id: UUID + tenant_id: UUID + plan: ProtectionPlan + status: string + started_at: DateTime +} + +Checkpoint *-> ProtectionPlan: stores a copy of + +class AutomaticOperation { + id: UUID + name: string + comments: string + tenant_id: UUID +} + +class PlanRetention extends AutomaticOperation { + daily: bool + weekly: bool + monthly: bool + yearly: bool + protection_plan: ProtectionPlan +} + +class ScheduledOperation <> extends AutomaticOperation { + trigger: Trigger +} + +ScheduledOperation *- Trigger: when should the operation should trigger + + +class BackupPlan extends ScheduledOperation { + protection_plan: ProtectionPlan +} + +BackupPlan *--> ProtectionPlan + +class DeleteCheckpoints extends ScheduledOperation { + query: string + protection_provider: ProtectionProvider +} + + +class ProtectionProvider { + name: string + description: string + managed_type: string + extended_info_schema: [ResourceType]JSONSchema + options_schema: [ResourceType]JSONSchema + checkpoints: []Checkpoint +} + +ProtectionProvider o-> Checkpoint: lists + +class ProtectionPlan { + id: UUID + is_enabled: boolean + name: string + comments: string + resources: []Resource + protection_provider: ProtectionProvider + parameters: dict + Start() + Suspend() +} + +ProtectionPlan "1" *--> "N" Resource: aggragates +ProtectionPlan -> ProtectionProvider + +class RestoreTarget { + keystone_uri: URI +} + +class Restoration { + template: string + target: RestoreTarget + provider: ProtectionProvider + checkpoint: Checkpoint + started_at: string +} + +Restoration *-> RestoreTarget: restores to + +@enduml +@startuml + +package "Plugin Examples" { + +interface BackupPlugin <> { + <> create_backup(resource: Resource, bank: Bank) + <> delete_backup(backup: Backup) +} + +interface ReplicationPlugin <> { + <> start_replication(resource: Resource, replicationType: eReplicationType) + <> create_snapshot_from_replicated_target(resource: Resource, replicationType: eReplicationType) + <> delete_snapshot(resource: Resource, replicationType: eReplicationType) + <> stop_replication(resource: Resource) +} + +ReplicationPlugin -> eReplicationType + +interface SnapshotPlugin <> { + <> create_snapshot(resource: Resource) + <> delete_snapshot(snapshot: Snapshot) +} + +enum eReplicationType { + hypervisor + cinder +} + +} + +@enduml diff --git a/doc/source/api/class_diagram.svg b/doc/source/api/class_diagram.svg new file mode 100644 index 00000000..e72834a5 --- /dev/null +++ b/doc/source/api/class_diagram.svg @@ -0,0 +1 @@ +"Smaug API model"Protectablename: stringinstances: []Resourceis_root: boolResourceid: UUIDtype: ResourceTypeschema: JSONSchemadependent_resources: []ResourceTriggerTimedTriggerEventTriggerCheckpointid: UUIDtenant_id: UUIDplan: ProtectionPlanstatus: stringstarted_at: DateTimeProtectionPlanid: UUIDis_enabled: booleanname: stringcomments: stringresources: []Resourceprotection_provider: ProtectionProviderparameters: dictStart()Suspend()AutomaticOperationid: UUIDname: stringcomments: stringtenant_id: UUIDPlanRetentiondaily: boolweekly: boolmonthly: boolyearly: boolprotection_plan: ProtectionPlan«abstract»ScheduledOperationtrigger: TriggerBackupPlanprotection_plan: ProtectionPlanDeleteCheckpointsquery: stringprotection_provider: ProtectionProviderProtectionProvidername: stringdescription: stringmanaged_type: stringextended_info_schema: [ResourceType]JSONSchemaoptions_schema: [ResourceType]JSONSchemacheckpoints: []CheckpointRestoreTargetkeystone_uri: URIRestorationtemplate: stringtarget: RestoreTargetprovider: ProtectionProvidercheckpoint: Checkpointstarted_at: stringlistsstores a copy ofwhen should the operation should triggerlistsaggragates1Nrestores to \ No newline at end of file diff --git a/doc/source/api/class_diagram_001.svg b/doc/source/api/class_diagram_001.svg new file mode 100644 index 00000000..e85c52b8 --- /dev/null +++ b/doc/source/api/class_diagram_001.svg @@ -0,0 +1 @@ +Plugin Examples«action set»BackupPlugin«protect» create_backup(resource: Resource, bank: Bank)«delete» delete_backup(backup: Backup)«action set»ReplicationPlugin«enable» start_replication(resource: Resource, replicationType: eReplicationType)«protect» create_snapshot_from_replicated_target(resource: Resource, replicationType: eReplicationType)«delete» delete_snapshot(resource: Resource, replicationType: eReplicationType)«disable» stop_replication(resource: Resource)eReplicationTypehypervisorcinder«action set»SnapshotPlugin«protect» create_snapshot(resource: Resource)«delete» delete_snapshot(snapshot: Snapshot) \ No newline at end of file diff --git a/doc/source/api/smaug_api.v1.yaml b/doc/source/api/smaug_api.v1.yaml new file mode 100644 index 00000000..be654131 --- /dev/null +++ b/doc/source/api/smaug_api.v1.yaml @@ -0,0 +1,547 @@ +swagger: '2.0' +info: + title: Smaug API + description: Protect all you hold dear + version: 0.99.0 +host: api.smaug.nowhere.com +schemes: + - https +basePath: /v1 +produces: + - application/json +paths: + /providers: + get: + summary: Providers + description: | + The Providers endpoint returns information about the providers + offered at a given service. All providers need to be configuered + first by the admin. + parameters: + - $ref: '#/parameters/nameFilterParam' + - $ref: '#/parameters/sortParam' + - $ref: '#/parameters/limitParam' + - $ref: '#/parameters/markerParam' + tags: + - Protection Provider + responses: + '200': + description: An array of providers + schema: + type: array + items: + $ref: '#/definitions/Provider' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + /providers/{provider_id}/checkpoints: + get: + summary: Checkpoints + description: | + The Checkpoints endpoint returns information about the checkpoints + offered at a given provider. + parameters: + - $ref: '#/parameters/provider_idParam' + - name: tenant_id + in: query + description: | + Specifies the ID of the tenant that owns this entity + type: string + format: uuid + required: false + - $ref: '#/parameters/sortParam' + - $ref: '#/parameters/limitParam' + - $ref: '#/parameters/markerParam' + tags: + - Protection Provider + - Checkpoint + responses: + '200': + description: An array of checkpoints + schema: + type: object + properties: + checkpoints: + type: array + items: + $ref: '#/definitions/Checkpoint' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + post: + summary: Checkpoints + description: | + The Checkpoints endpoint creates a checkpoint + at a given provider. + parameters: + - $ref: '#/parameters/provider_idParam' + - name: checkpoint + in: body + required: true + schema: + $ref: '#/definitions/Checkpoint' + tags: + - Protection Provider + - Checkpoint + responses: + '200': + description: Checkpoint created + schema: + $ref: '#/definitions/Checkpoint' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + /providers/{provider_id}/checkpoints/{checkpoint_id}: + delete: + summary: Checkpoints + description: | + The Checkpoints endpoint deletes a checkpoint + at a given provider. + parameters: + - $ref: '#/parameters/provider_idParam' + - $ref: '#/parameters/checkpoint_idParam' + tags: + - Protection Provider + - Checkpoint + responses: + '200': + description: Checkpoint deleted + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + /{tenant_id}/plans: + get: + summary: Get protection plans + description: | + The Plans endpoint returns information about the protection plans + offered for the given tenant. + parameters: + - $ref: '#/parameters/tenantParam' + - $ref: '#/parameters/nameFilterParam' + - $ref: '#/parameters/sortParam' + - $ref: '#/parameters/limitParam' + - $ref: '#/parameters/markerParam' + tags: + - Tenant API + - Protection Plan + responses: + '200': + description: An array of protection plans + schema: + type: object + properties: + plans: + type: array + items: + $ref: '#/definitions/ProtectionPlan' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + post: + summary: Create a plan + description: | + Create a new plan. The update will create a new revision for + the plan. + tags: + - Tenant API + - Protection Plan + parameters: + - $ref: '#/parameters/tenantParam' + - name: plan + in: body + required: true + schema: + $ref: '#/definitions/ProtectionPlan' + responses: + '200': + description: The new checkpoint + schema: + $ref: '#/definitions/ProtectionPlan' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + /{tenant_id}/plans/{plan_id}: + get: + summary: Protection Plan + description: | + The Plan endpoint returns information about a specific plan. + parameters: + - $ref: '#/parameters/tenantParam' + - $ref: '#/parameters/plan_idParam' + tags: + - Tenant API + - Protection Plan + responses: + '200': + description: the protection plan + schema: + $ref: '#/definitions/ProtectionPlan' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + delete: + summary: Protection Plan + description: | + The Plan endpoint deletes a specific plan. + parameters: + - $ref: '#/parameters/tenantParam' + - $ref: '#/parameters/plan_idParam' + tags: + - Tenant API + - Protection Plan + responses: + '200': + description: the protection plan + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + /resource_types: + get: + summary: Resource Types + description: | + The returns all the available resource types. + parameters: + - $ref: '#/parameters/nameFilterParam' + - $ref: '#/parameters/sortParam' + - $ref: '#/parameters/limitParam' + - $ref: '#/parameters/markerParam' + + tags: + - Resource + responses: + '200': + description: The available resource types + schema: + type: object + properties: + resource_types: + type: array + items: + $ref: '#/definitions/ResourceTypeInfo' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + /resource_types/{resource_type}: + get: + summary: Resource Type + description: | + The returns all the available resource types. + parameters: + - $ref: '#/parameters/resource_typeParam' + tags: + - Resource + responses: + '200': + description: The resource type + schema: + $ref: '#/definitions/ResourceTypeInfo' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + /resource_types/{resource_type}/instances: + get: + summary: Resource Instances + description: | + The returns all the available instances for the resource type. + parameters: + - $ref: '#/parameters/resource_typeParam' + - $ref: '#/parameters/nameFilterParam' + - $ref: '#/parameters/sortParam' + - $ref: '#/parameters/limitParam' + - $ref: '#/parameters/markerParam' + + tags: + - Resource + responses: + '200': + description: The available instances for the resource type. + schema: + type: object + properties: + resources: + type: array + items: + $ref: '#/definitions/Resource' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + /{tenant_id}/scheduled_operations: + get: + summary: Scheduled Operations + description: | + Scheduled operations are operations that will be executed when + a specific trigger is triggered. + parameters: + - $ref: '#/parameters/tenantParam' + - $ref: '#/parameters/nameFilterParam' + - $ref: '#/parameters/sortParam' + - $ref: '#/parameters/limitParam' + - $ref: '#/parameters/markerParam' + tags: + - Tenant API + - Scheduled Operation + responses: + '200': + description: An array of scheduled operations + schema: + type: object + properties: + scheduled_operations: + type: array + items: + $ref: '#/definitions/ScheduledOperation' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + post: + summary: Create an scheduled operation + description: | + Create a new scheduled operation. + tags: + - Tenant API + - Scheduled Operation + parameters: + - $ref: '#/parameters/tenantParam' + - name: schedlued_operation + in: body + required: true + schema: + $ref: '#/definitions/ScheduledOperation' + responses: + '200': + description: The new scheduled operation + schema: + $ref: '#/definitions/ScheduledOperation' + default: + description: Unexpected error + schema: + $ref: '#/definitions/Error' + +definitions: + Provider: + type: object + required: [ name ] + properties: + id: + readOnly: true + type: string + format: UUID + description: Unique identifier representing a specific protection provider. + description: + type: string + description: Description of provider. + name: + type: string + description: Display name of provider. + Checkpoint: + type: object + properties: + id: + readOnly: true + type: string + format: UUID + description: Unique identifier representing a specific protection provider. + description: + type: string + description: Description of provider. + Resource: + type: object + properties: + type: + $ref: '#/definitions/ResourceType' + readOnly: true + id: + readOnly: true + type: string + ProtectionPlan: + type: object + required: [ name, protection_provider_id, resources ] + properties: + id: + readOnly: true + type: string + format: UUID + description: Unique identifier representing a specific protection protection plan. + name: + type: string + description: Display name of plan. + comments: + type: string + description: Comments about the plan. + resources: + type: array + items: + $ref: '#/definitions/Resource' + protection_provider_id: + type: string + format: UUID + description: | + Unique identifier representing a specific protection provider that + will store checkpoints for this protection plan. + parameters: + type: object + description: TODO + ResourceType: + type: string + format: Heat Type String + example: "OS::Nova::Server" + description: | + Name of the resource type. When available the types that are defined by Heat + are used. + ResourceTypeInfo: + type: object + properties: + name: + $ref: '#/definitions/ResourceType' + is_root: + type: boolean + description: | + Defines whether this type has any dependencies or not. Useful for + UIs. + dependent_types: + type: array + description: | + List of types that might depend on this type. For example an + "OS::Nova::Server" has "OS::Cinder::Volume" as a dependent type. + items: + $ref: '#/definitions/ResourceType' + OperationDefinition: + type: object + discriminator: type + required: [ type ] + properties: + id: + type: string + format: UUID + description: | + Unique identifier representing a specific operation definition. + type: + type: string + description: | + Type of the operation. This defines what kind of operation this + object defines the arguments for. + ProtectOperationDefinition: + description: | + Opration definition for protect operation. + allOf: + - $ref: '#/definitions/OperationDefinition' + - type: object + properties: + protection_plan_id: + type: string + format: UUID + required: [ protection_plan_id ] + ScheduledOperation: + type: object + properties: + id: + type: string + format: UUID + description: | + Unique identifier representing a specific scheduled operation. + name: + type: string + description: Display name of scheduled operation. + comments: + type: string + description: Comments about the scheduled operation. + trigger_id: + type: string + format: UUID + operation_definition: + $ref: '#/definitions/OperationDefinition' + + Error: + type: object + properties: + code: + type: integer + format: int32 + message: + type: string + fields: + type: string +parameters: + sortParam: + name: sort + in: query + description: | + Comma-separated list of sort keys and optional sort directions in the + form of '<key>[:<direction>]`. A valid direction is asc + (ascending) or desc (descending). + required: false + type: string + limitParam: + name: limit + in: query + description: | + Requests a specified page size of returned items from the query. + Returns a number of items up to the specified limit value. + Use the limit parameter to make an initial limited request and use the + ID of the last-seen item from the response as the marker parameter value + in a subsequent limited request. + type: integer + format: int64 + markerParam: + name: marker + in: query + description: | + Specifies the ID of the last-seen item. Use the limit parameter to make + an initial limited request and use the ID of the last-seen item from the + response as the marker parameter value in a subsequent limited request. + type: string + tenantParam: + name: tenant_id + in: path + description: | + Specifies the ID of the tenant that owns this entity + type: string + format: uuid + required: true + nameFilterParam: + name: name + in: query + format: regex + description: name of the entity. Could be a regex pattern. + required: false + type: string + provider_idParam: + name: provider_id + type: string + format: uuid + required: true + in: path + description: id of the provider. + plan_idParam: + name: plan_id + type: string + format: uuid + required: true + in: path + description: id of the plan. + checkpoint_idParam: + name: checkpoint_id + type: string + format: uuid + required: true + in: path + description: id of the checkpoint. + resource_typeParam: + name: resource_type + type: string + format: Heat Type String + required: true + in: path + description: the resource type.