From 3a2d220f8a11570a90fc2cfcdb111fe577e4b800 Mon Sep 17 00:00:00 2001 From: Kiran Pawar Date: Wed, 16 Aug 2023 16:36:06 +0000 Subject: [PATCH] Update Share backup APIs and add api ref - Follow up change to fix suggestions from earlier pull request i.e. https://review.opendev.org/c/openstack/manila/+/343980 . - Add API-ref docs - Rename column availability_zone to availability_zone_id in share_backups table. Implement: blueprint share-backup Closes-bug: #2031311 Change-Id: Ice01ab7892b1eb52b3202f2c79957977f73f3aca --- api-ref/source/index.rst | 1 + api-ref/source/limits.inc | 4 + api-ref/source/parameters.yaml | 218 +++++++- api-ref/source/quota-classes.inc | 8 + api-ref/source/quota-sets.inc | 15 + api-ref/source/samples/limits-response.json | 6 +- .../samples/quota-classes-show-response.json | 4 +- .../quota-classes-update-response.json | 4 +- .../samples/quota-show-detail-response.json | 9 +- .../source/samples/quota-show-response.json | 4 +- .../source/samples/quota-update-response.json | 4 +- .../samples/share-backup-create-request.json | 8 + .../samples/share-backup-create-response.json | 15 + .../share-backup-reset-status-request.json | 5 + .../samples/share-backup-restore-request.json | 3 + .../share-backup-restore-response.json | 6 + .../samples/share-backup-show-response.json | 15 + .../samples/share-backup-update-request.json | 6 + .../samples/share-backup-update-response.json | 15 + .../share-backups-list-detailed-response.json | 30 ++ .../samples/share-backups-list-response.json | 16 + .../shares-list-detailed-response.json | 2 + api-ref/source/share-backups.inc | 477 ++++++++++++++++++ api-ref/source/shares.inc | 9 + devstack/plugin.sh | 8 + manila/api/v1/share_unmanage.py | 7 + manila/api/v2/share_backups.py | 38 +- manila/api/views/share_backups.py | 15 +- manila/api/views/shares.py | 5 + manila/data/manager.py | 57 ++- .../2d708a9a3ba9_backup_change_az_to_az_id.py | 94 ++++ manila/db/sqlalchemy/api.py | 2 + manila/db/sqlalchemy/models.py | 20 +- manila/policies/share_backup.py | 43 ++ manila/share/api.py | 11 +- manila/share/manager.py | 10 +- manila/tests/api/v1/test_share_unmanage.py | 36 ++ manila/tests/api/v2/test_share_backups.py | 28 +- manila/tests/api/v2/test_shares.py | 4 + manila/tests/data/test_manager.py | 21 +- manila/tests/db/sqlalchemy/test_api.py | 10 - manila/tests/fake_share.py | 1 - 42 files changed, 1224 insertions(+), 70 deletions(-) create mode 100644 api-ref/source/samples/share-backup-create-request.json create mode 100644 api-ref/source/samples/share-backup-create-response.json create mode 100644 api-ref/source/samples/share-backup-reset-status-request.json create mode 100644 api-ref/source/samples/share-backup-restore-request.json create mode 100644 api-ref/source/samples/share-backup-restore-response.json create mode 100644 api-ref/source/samples/share-backup-show-response.json create mode 100644 api-ref/source/samples/share-backup-update-request.json create mode 100644 api-ref/source/samples/share-backup-update-response.json create mode 100644 api-ref/source/samples/share-backups-list-detailed-response.json create mode 100644 api-ref/source/samples/share-backups-list-response.json create mode 100644 api-ref/source/share-backups.inc create mode 100644 manila/db/migrations/alembic/versions/2d708a9a3ba9_backup_change_az_to_az_id.py diff --git a/api-ref/source/index.rst b/api-ref/source/index.rst index 100770fa9c..4c200453dc 100644 --- a/api-ref/source/index.rst +++ b/api-ref/source/index.rst @@ -66,3 +66,4 @@ Shared File Systems API (EXPERIMENTAL) .. include:: experimental.inc .. include:: share-migration.inc .. include:: share-server-migration.inc +.. include:: share-backups.inc diff --git a/api-ref/source/limits.inc b/api-ref/source/limits.inc index 764240dc60..35b319320d 100644 --- a/api-ref/source/limits.inc +++ b/api-ref/source/limits.inc @@ -72,6 +72,8 @@ Response parameters - maxTotalShareNetworks: maxTotalShareNetworks - maxTotalShareReplicas: maxTotalShareReplicas - maxTotalReplicaGigabytes: maxTotalReplicaGigabytes + - maxTotalShareBackups: maxTotalShareBackups + - maxTotalBackupGigabytes: maxTotalBackupGigabytes - totalSharesUsed: totalSharesUsed - totalShareSnapshotsUsed: totalShareSnapshotsUsed - totalShareNetworksUsed: totalShareNetworksUsed @@ -79,6 +81,8 @@ Response parameters - totalSnapshotGigabytesUsed: totalSnapshotGigabytesUsed - totalShareReplicasUsed: totalShareReplicasUsed - totalReplicaGigabytesUsed: totalReplicaGigabytesUsed + - totalShareBackupsUsed: totalShareBackupsUsed + - totalBackupGigabytesUsed: totalBackupGigabytesUsed - uri: uri - regex: regex - value: value diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index a91b7470ba..42bca669cb 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -25,6 +25,12 @@ api_version: type: string description: > The API version as returned in the links from the ``GET /`` call. +backup_id_request_path: + description: | + The UUID of the share backup. + in: path + required: true + type: string export_location_id_path: description: | The UUID of the export location. @@ -224,6 +230,34 @@ backend_query: in: query required: false type: string +backup_host_query: + description: | + The host name of the backup to query with. Querying by hostname is a + privileged operation. If restricted by API policy, this query parameter + may be silently ignored by the server. + in: query + required: false + type: string +backup_share_id_query: + description: | + The UUID of the share that the backup pertains to. + in: query + required: false + type: string +backup_status_query: + description: | + Filters by a backup status. A valid filter value can be one of 'creating', + 'error', 'available', 'restoring'. + in: query + required: false + type: string +backup_topic_query: + description: | + Filters by a backup topic. A valid filter value can be one of + 'manila-data', 'manila-share'. + in: query + required: false + type: string cidr_query: description: | The CIDR to filter share networks. @@ -292,6 +326,12 @@ description_inexact_query: required: false type: string min_version: 2.36 +description_inexact_query_versionless: + description: | + The description pattern that can be used to filter share backups. + in: query + required: false + type: string description_query: description: | The user defined description text that can be used to filter resources. @@ -386,7 +426,7 @@ is_soft_deleted_query: min_version: 2.69 limit: description: | - The maximum number of shares to return. + The maximum number of resource records to return. in: query required: false type: integer @@ -417,6 +457,12 @@ name_inexact_query: required: false type: string min_version: 2.36 +name_inexact_query_versionless: + description: | + The name pattern that can be used to filter share backups. + in: query + required: false + type: string name_query: description: | The user defined name of the resource to filter resources by. @@ -688,6 +734,15 @@ sort_key: in: query required: false type: string +sort_key_backup: + description: | + The key to sort a list of share backups. A valid value + is ``id``, ``status``, ``size``, ``host``, ``share_id`` + ``availability_zone``, ``created_at``, ``updated_at``, ``display_name``, + ``topic``, ``progress`` and ``restore_progress`` + in: query + required: false + type: string sort_key_messages: description: | The key to sort a list of messages. A valid value @@ -715,6 +770,12 @@ sort_key_transfer: in: query required: false type: string +source_backup_id_query: + description: | + The UUID of the share's backup to filter the request based on. + in: query + required: false + type: string source_share_group_snapshot_id_query: description: | The source share group snapshot ID to list the @@ -987,6 +1048,63 @@ backend_name: in: body required: true type: string +backup_az: + description: | + The availability zone. + in: body + required: true + type: string +backup_id_response: + description: | + The UUID of the share backup. + in: body + required: true + type: string +backup_options_request: + description: | + One or more backup options key and value pairs as a + url encoded dictionary of strings. + in: body + required: false + type: object +backup_progress: + description: | + The progress of the backup creation in percentange. + in: body + required: true + type: string +backup_restore_progress: + description: | + The progress of the backup restoration in percentage. + in: body + required: true + type: string +backup_share_id: + description: | + The UUID of the share that the backup pertains to. + in: body + required: true + type: string +backup_size: + description: | + The share backup size, in GiBs. + in: body + required: true + type: integer +backup_status: + description: | + The status of backup which can be one of ``creating``, ``error``, + ``available``, ``restoring``. + in: body + required: true + type: string +backup_status_request: + description: | + The backup status, which can be ``available``, + ``error``, ``creating``, ``deleting``, ``restoring``. + in: body + required: false + type: string capabilities: description: | The back end capabilities which include ``qos``, ``total_capacity_gb``, @@ -1655,6 +1773,22 @@ managed_share_user_id: required: true type: string min_version: 2.16 +maxTotalBackupGigabytes: + description: | + The total maximum number of backup gigabytes + that are allowed in a project. + in: body + required: true + type: integer + min_version: 2.80 +maxTotalBackupGigabytesOptional: + description: | + The total maximum number of backup gigabytes + that are allowed in a project. + in: body + required: false + type: integer + min_version: 2.80 maxTotalReplicaGigabytes: description: | The maximum number of replica gigabytes that are allowed in a project. @@ -1673,6 +1807,22 @@ maxTotalReplicaGigabytesOptional: required: false type: integer min_version: 2.53 +maxTotalShareBackups: + description: | + The total maximum number of share backups that + are allowed in a project. + in: body + required: true + type: integer + min_version: 2.80 +maxTotalShareBackupsOptional: + description: | + The total maximum number of share backups that + are allowed in a project. + in: body + required: false + type: integer + min_version: 2.80 maxTotalShareGigabytes: description: | The total maximum number of share gigabytes that @@ -2161,6 +2311,51 @@ protocol: in: body required: true type: string +quota_backup_gigabytes: + description: | + The number of gigabytes for the backups allowed for each project. + in: body + min_version: 2.80 + required: true + type: integer +quota_backup_gigabytes_detail: + description: | + The limit, in_use, reserved number of gigabytes for the + backups allowed for each project. + in: body + min_version: 2.80 + required: true + type: object +quota_backup_gigabytes_request: + description: | + The number of gigabytes for the backups for the + project. + in: body + min_version: 2.80 + required: false + type: integer +quota_backups: + description: | + The number of backups allowed for each project. + in: body + min_version: 2.80 + required: true + type: integer +quota_backups_detail: + description: | + The limit, in_use, reserved number of backups allowed + for each project. + in: body + min_version: 2.80 + required: true + type: object +quota_backups_request: + description: | + The number of backups for the project. + in: body + min_version: 2.80 + required: false + type: integer quota_class_id: description: | A ``quota_class_set`` id. @@ -3703,6 +3898,13 @@ snapshot_user_id: required: true type: string min_version: 2.17 +source_backup_id_shares_response: + description: | + The UUID of the backup that was restored in the share. + in: body + required: true + type: string + min_version: 2.80 source_share_group_snapshot_id: description: | The source share group snapshot ID to create the @@ -3805,6 +4007,13 @@ total_progress_server_migration: in: body required: true type: integer +totalBackupGigabytesUsed: + description: | + The total number of gigabytes used in a project + by backups. + in: body + required: true + type: integer totalReplicaGigabytesUsed: description: | The total number of replica gigabytes used in a @@ -3812,6 +4021,13 @@ totalReplicaGigabytesUsed: in: body required: true type: integer +totalShareBackupsUsed: + description: | + The total number of created share backups in a + project. + in: body + required: true + type: integer totalShareGigabytesUsed: description: | The total number of gigabytes used in a project diff --git a/api-ref/source/quota-classes.inc b/api-ref/source/quota-classes.inc index 3703d8e3b3..6ca35b6291 100644 --- a/api-ref/source/quota-classes.inc +++ b/api-ref/source/quota-classes.inc @@ -10,6 +10,8 @@ Quota classes can be shown and updated for a project. APIs in API version 2.53. Per share gigabytes was added to quota management APIs in API version 2.62. + Share backups and backup gigabytes were added to quota management + APIs in API version 2.80. Show quota classes for a project @@ -58,6 +60,8 @@ Response Parameters - share_replicas: maxTotalShareReplicas - replica_gigabytes: maxTotalReplicaGigabytes - per_share_gigabytes: perShareGigabytes + - backups: maxTotalShareBackups + - backup_gigabytes: maxTotalBackupGigabytes Response Example ---------------- @@ -102,6 +106,8 @@ Request - share-replicas: maxTotalShareReplicasOptional - replica-gigabytes: maxTotalReplicaGigabytesOptional - per-share-gigabytes: perShareGigabytesOptional + - backups: maxTotalShareBackupsOptional + - backup-gigabytes: maxTotalBackupGigabytesOptional Request Example --------------- @@ -126,6 +132,8 @@ Response Parameters - share_replicas: maxTotalShareReplicas - replica_gigabytes: maxTotalReplicaGigabytes - per_share_gigabytes: perShareGigabytes + - backups: maxTotalShareBackups + - backup_gigabytes: maxTotalBackupGigabytes Response Example ---------------- diff --git a/api-ref/source/quota-sets.inc b/api-ref/source/quota-sets.inc index 949a47cc76..f5222cdd9b 100644 --- a/api-ref/source/quota-sets.inc +++ b/api-ref/source/quota-sets.inc @@ -22,6 +22,8 @@ Provides quotas management support. - ``share_replicas`` (since API version 2.53) - ``replica_gigabytes`` (since API version 2.53) - ``per_share_gigabytes`` (since API version 2.62) + - ``backups`` (since API version 2.80) + - ``backup_gigabytes`` (since API version 2.80) In order to manipulate share type quotas, the requests will be similar to the examples below, except that the ``user_id={user_id}`` must be @@ -36,6 +38,9 @@ Provides quotas management support. Per share gigabytes was added to quota management APIs in API version 2.62. + Share backups and backup gigabytes were added to quota management + APIs in API version 2.80. + Show default quota set ~~~~~~~~~~~~~~~~~~~~~~ @@ -83,6 +88,8 @@ Response parameters - share_replicas: quota_share_replicas - replica_gigabytes: quota_replica_gigabytes - per_share_gigabytes: quota_per_share_gigabytes + - backups: quota_backups + - backup_gigabytes: quota_backup_gigabytes Response example ---------------- @@ -142,6 +149,8 @@ Response parameters - share_replicas: quota_share_replicas - replica_gigabytes: quota_replica_gigabytes - per_share_gigabytes: quota_per_share_gigabytes + - backups: quota_backups + - backup_gigabytes: quota_backup_gigabytes Response example ---------------- @@ -203,6 +212,8 @@ Response parameters - share_replicas: quota_share_replicas_detail - replica_gigabytes: quota_replica_gigabytes_detail - per_share_gigabytes: quota_per_share_gigabytes_detail + - backups: quota_backups_detail + - backup_gigabytes: quota_backup_gigabytes_detail Response example ---------------- @@ -256,6 +267,8 @@ Request - share_replicas: quota_share_replicas_request - replica_gigabytes: quota_replica_gigabytes_request - per_share_gigabytes: quota_per_share_gigabytes_request + - backups: quota_backups_request + - backup_gigabytes: quota_backup_gigabytes_request Request example --------------- @@ -280,6 +293,8 @@ Response parameters - share_replicas: quota_share_replicas - replica_gigabytes: quota_replica_gigabytes - per_share_gigabytes: quota_per_share_gigabytes + - backups: quota_backups + - backup_gigabytes: quota_backup_gigabytes Response example ---------------- diff --git a/api-ref/source/samples/limits-response.json b/api-ref/source/samples/limits-response.json index e716cd51d5..3764adf34c 100644 --- a/api-ref/source/samples/limits-response.json +++ b/api-ref/source/samples/limits-response.json @@ -15,7 +15,11 @@ "maxTotalShareReplicas": 100, "maxTotalReplicaGigabytes": 1000, "totalShareReplicasUsed": 0, - "totalReplicaGigabytesUsed": 0 + "totalReplicaGigabytesUsed": 0, + "maxTotalShareBackups": 100, + "maxTotalBackupGigabytes": 1000, + "totalShareBackupsUsed": 0, + "totalBackupGigabytesUsed": 0 } } } diff --git a/api-ref/source/samples/quota-classes-show-response.json b/api-ref/source/samples/quota-classes-show-response.json index 54d4242b55..ce5f5fabfb 100644 --- a/api-ref/source/samples/quota-classes-show-response.json +++ b/api-ref/source/samples/quota-classes-show-response.json @@ -10,6 +10,8 @@ "share_networks": 10, "share_replicas": 100, "replica_gigabytes": 1000, - "per_share_gigabytes": -1 + "per_share_gigabytes": -1, + "backups": 50, + "backup_gigabytes": 1000 } } diff --git a/api-ref/source/samples/quota-classes-update-response.json b/api-ref/source/samples/quota-classes-update-response.json index 87fe1db420..4f8c0b9384 100644 --- a/api-ref/source/samples/quota-classes-update-response.json +++ b/api-ref/source/samples/quota-classes-update-response.json @@ -9,6 +9,8 @@ "share_networks": 10, "share_replicas": 100, "replica_gigabytes": 1000, - "per_share_gigabytes": -1 + "per_share_gigabytes": -1, + "backups": 50, + "backup_gigabytes": 1000 } } diff --git a/api-ref/source/samples/quota-show-detail-response.json b/api-ref/source/samples/quota-show-detail-response.json index 53b3984b35..12ab9f92a2 100644 --- a/api-ref/source/samples/quota-show-detail-response.json +++ b/api-ref/source/samples/quota-show-detail-response.json @@ -30,7 +30,12 @@ "reserved": 0}, "per_share_gigabytes": {"in_use": 0, "limit": -1, - "reserved": 0} - + "reserved": 0}, + "backup_gigabytes": {"in_use": 0, + "limit": 1000, + "reserved": 0}, + "backups": {"in_use": 0, + "limit": 50, + "reserved": 0} } } diff --git a/api-ref/source/samples/quota-show-response.json b/api-ref/source/samples/quota-show-response.json index 6f985a11b6..f322ec0ee8 100644 --- a/api-ref/source/samples/quota-show-response.json +++ b/api-ref/source/samples/quota-show-response.json @@ -10,6 +10,8 @@ "share_group_snapshots": 10, "share_replicas": 100, "replica_gigabytes": 1000, - "per_share_gigabytes": -1 + "per_share_gigabytes": -1, + "backups": 50, + "backup_gigabytes": 1000 } } diff --git a/api-ref/source/samples/quota-update-response.json b/api-ref/source/samples/quota-update-response.json index 29e73d8a9d..9b6e6a10c6 100644 --- a/api-ref/source/samples/quota-update-response.json +++ b/api-ref/source/samples/quota-update-response.json @@ -9,6 +9,8 @@ "share_group_snapshots": 12, "share_replicas": 89, "replica_gigabytes": 1000, - "per_share_gigabytes": -1 + "per_share_gigabytes": -1, + "backups": 40, + "backup_gigabytes": 500 } } diff --git a/api-ref/source/samples/share-backup-create-request.json b/api-ref/source/samples/share-backup-create-request.json new file mode 100644 index 0000000000..dc403cfe05 --- /dev/null +++ b/api-ref/source/samples/share-backup-create-request.json @@ -0,0 +1,8 @@ +{ + "share_backup": { + "share_id": "7b11dd53-546e-43cd-af0e-875434238c30", + "backup_options": {}, + "description": null, + "name": "backup1" + } +} diff --git a/api-ref/source/samples/share-backup-create-response.json b/api-ref/source/samples/share-backup-create-response.json new file mode 100644 index 0000000000..7d95a9754c --- /dev/null +++ b/api-ref/source/samples/share-backup-create-response.json @@ -0,0 +1,15 @@ +{ + "share_backup": { + "id": "c1cdc0ce-4ddc-4018-9796-505d2e26fcc7", + "share_id": "7b11dd53-546e-43cd-af0e-875434238c30", + "status": "creating", + "name": "backup1", + "description": null, + "size": 1, + "created_at": "2023-08-16T13:03:59.020692", + "updated_at": "2023-08-16T13:03:59.020692", + "availability_zone": null, + "progress": "0", + "restore_progress": "0" + } +} diff --git a/api-ref/source/samples/share-backup-reset-status-request.json b/api-ref/source/samples/share-backup-reset-status-request.json new file mode 100644 index 0000000000..e5f70b3722 --- /dev/null +++ b/api-ref/source/samples/share-backup-reset-status-request.json @@ -0,0 +1,5 @@ +{ + "reset_status": { + "status": "error" + } +} diff --git a/api-ref/source/samples/share-backup-restore-request.json b/api-ref/source/samples/share-backup-restore-request.json new file mode 100644 index 0000000000..d38291fe08 --- /dev/null +++ b/api-ref/source/samples/share-backup-restore-request.json @@ -0,0 +1,3 @@ +{ + "restore": null +} diff --git a/api-ref/source/samples/share-backup-restore-response.json b/api-ref/source/samples/share-backup-restore-response.json new file mode 100644 index 0000000000..b8d1bf1380 --- /dev/null +++ b/api-ref/source/samples/share-backup-restore-response.json @@ -0,0 +1,6 @@ +{ + "restore": { + "backup_id": "c1cdc0ce-4ddc-4018-9796-505d2e26fcc7", + "share_id": "7b11dd53-546e-43cd-af0e-875434238c30" + } +} diff --git a/api-ref/source/samples/share-backup-show-response.json b/api-ref/source/samples/share-backup-show-response.json new file mode 100644 index 0000000000..b1e35bc61f --- /dev/null +++ b/api-ref/source/samples/share-backup-show-response.json @@ -0,0 +1,15 @@ +{ + "share_backup": { + "id": "c1cdc0ce-4ddc-4018-9796-505d2e26fcc7", + "share_id": "7b11dd53-546e-43cd-af0e-875434238c30", + "status": "available", + "name": "backup1", + "description": null, + "size": 1, + "created_at": "2023-08-16T13:03:59.000000", + "updated_at": "2023-08-16T13:04:15.000000", + "availability_zone": null, + "progress": "100", + "restore_progress": "0" + } +} diff --git a/api-ref/source/samples/share-backup-update-request.json b/api-ref/source/samples/share-backup-update-request.json new file mode 100644 index 0000000000..57be3a297a --- /dev/null +++ b/api-ref/source/samples/share-backup-update-request.json @@ -0,0 +1,6 @@ +{ + "share_backup": { + "display_name": "backup2", + "display_description": "I am changing a description also. Here is a backup" + } +} diff --git a/api-ref/source/samples/share-backup-update-response.json b/api-ref/source/samples/share-backup-update-response.json new file mode 100644 index 0000000000..778babfb17 --- /dev/null +++ b/api-ref/source/samples/share-backup-update-response.json @@ -0,0 +1,15 @@ +{ + "share_backup": { + "id": "fa32a89f-ed0f-4906-b1d7-92eedf98fbb5", + "share_id": "7b11dd53-546e-43cd-af0e-875434238c30", + "status": "available", + "name": "backup2", + "description": "I am changing a description also. Here is a backup", + "size": 1, + "created_at": "2023-08-16T13:18:55.000000", + "updated_at": "2023-08-16T13:33:15.000000", + "availability_zone": null, + "progress": "100", + "restore_progress": "0" + } +} diff --git a/api-ref/source/samples/share-backups-list-detailed-response.json b/api-ref/source/samples/share-backups-list-detailed-response.json new file mode 100644 index 0000000000..15aa881977 --- /dev/null +++ b/api-ref/source/samples/share-backups-list-detailed-response.json @@ -0,0 +1,30 @@ +{ + "share_backups": [ + { + "id": "1125c47a-0216-4ee0-a517-0460d63301a6", + "share_id": "112dffd-f033-4248-a315-319ca2bd70c8", + "status": "available", + "name": "backup3", + "description": null, + "size": 1, + "created_at": "2023-08-16T12:34:57.000000", + "updated_at": "2023-08-17T12:14:15.000000", + "availability_zone": null, + "progress": "100", + "restore_progress": "0" + }, + { + "id": "c1cdc0ce-4ddc-4018-9796-505d2e26fcc7", + "share_id": "7b11dd53-546e-43cd-af0e-875434238c30", + "status": "creating", + "name": "backup1", + "description": null, + "size": 1, + "created_at": "2023-08-16T13:03:59.020692", + "updated_at": "2023-08-16T13:13:15.000002", + "availability_zone": null, + "progress": "0", + "restore_progress": "0" + } + ] +} diff --git a/api-ref/source/samples/share-backups-list-response.json b/api-ref/source/samples/share-backups-list-response.json new file mode 100644 index 0000000000..ce0d991dde --- /dev/null +++ b/api-ref/source/samples/share-backups-list-response.json @@ -0,0 +1,16 @@ +{ + "share_backups": [ + { + "id": "1125c47a-0216-4ee0-a517-0460d63301a6", + "name": "backup3", + "share_id": "112dffd-f033-4248-a315-319ca2bd70c8", + "status": "available" + }, + { + "id": "c1cdc0ce-4ddc-4018-9796-505d2e26fcc7", + "name": "backup1", + "share_id": "7b11dd53-546e-43cd-af0e-875434238c30", + "status": "creating" + } + ] +} diff --git a/api-ref/source/samples/shares-list-detailed-response.json b/api-ref/source/samples/shares-list-detailed-response.json index d8bf097f28..626b091a19 100644 --- a/api-ref/source/samples/shares-list-detailed-response.json +++ b/api-ref/source/samples/shares-list-detailed-response.json @@ -17,6 +17,7 @@ "share_server_id": "87d8943a-f5da-47a4-b2f2-ddfa6794aa82", "share_group_id": null, "snapshot_id": null, + "source_backup_id": null, "id": "f45cc5b2-d1bb-4a3e-ba5b-5c4125613adc", "size": 1, "share_type": "25747776-08e5-494f-ab40-a64b9d20d8f7", @@ -58,6 +59,7 @@ ], "share_server_id": "87d8943a-f5da-47a4-b2f2-ddfa6794aa82", "snapshot_id": null, + "source_backup_id": null, "id": "c4a2ced4-2c9f-4ae1-adaa-6171833e64df", "size": 1, "share_type": "25747776-08e5-494f-ab40-a64b9d20d8f7", diff --git a/api-ref/source/share-backups.inc b/api-ref/source/share-backups.inc new file mode 100644 index 0000000000..c5acf435eb --- /dev/null +++ b/api-ref/source/share-backups.inc @@ -0,0 +1,477 @@ +.. -*- rst -*- + +Share backups (since API v2.80) +=============================== + +Use the Shared File Systems service to make backups of shares. A share +backup is a point-in-time, read-only copy of the data that is +contained in a share. The APIs below allow controlling share backups. They +are represented by a "backup" resource in the Shared File Systems service, +and they can have user-defined metadata such as a name and description. + +You can create, restore, update, list and delete share backups. After you +create a share backup, you can access backup and use it. You can also restore +a backup into a share as long as certain criteria are met e.g. size. + +You can update a share backup to change its name or description. As +administrator, you can also reset the state of a backup. Backup can be in +one of the following states: + +- ``available`` + +- ``error`` + +- ``creating`` + +- ``deleting`` + +- ``restoring`` + + +During a backup or restore operation, share can be in one of the following +states: + +- ``available`` + +- ``backup_creating`` + +- ``backup_restoring`` + +- ``backup_restoring_error`` + + +List share backups +~~~~~~~~~~~~~~~~~~ + +.. rest_method:: GET /v2/share-backups + +.. versionadded:: 2.80 + +Lists all share backups. + +Response codes +-------------- + +.. rest_status_code:: success status.yaml + + - 200 + +.. rest_status_code:: error status.yaml + + - 400 + - 401 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - share_id: backup_share_id_query + - name~: name_inexact_query_versionless + - description~: description_inexact_query_versionless + - limit: limit + - offset: offset + - sort_key: sort_key_backup + - sort_dir: sort_dir + - status: backup_status_query + - host: backup_host_query + - topic: backup_topic_query + +Response parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - id: backup_id_response + - share_id: backup_share_id + - status: backup_status + +Response example +---------------- + +.. literalinclude:: samples/share-backups-list-response.json + :language: javascript + + +List share backups with details +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: GET /v2/share-backups/detail + +.. versionadded:: 2.80 + +Lists all share backups with details. + +Response codes +-------------- + +.. rest_status_code:: success status.yaml + + - 200 + +.. rest_status_code:: error status.yaml + + - 400 + - 401 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - share_id: backup_share_id_query + - name~: name_inexact_query_versionless + - description~: description_inexact_query_versionless + - limit: limit + - offset: offset + - sort_key: sort_key_backup + - sort_dir: sort_dir + - status: backup_status_query + - host: backup_host_query + - topic: backup_topic_query + +Response parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - id: backup_id_response + - share_id: backup_share_id + - status: backup_status + - size: backup_size + - availability_zone: backup_az + - name: name + - description: description + - created_at: created_at + - updated_at: updated_at + - progress: backup_progress + - restore_progress: backup_restore_progress + +Response example +---------------- + +.. literalinclude:: samples/share-backups-list-detailed-response.json + :language: javascript + + +Show share backup details +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: GET /v2/share-backups/{backup_id} + +.. versionadded:: 2.80 + +Shows details for a share backup. + +Response codes +-------------- + +.. rest_status_code:: success status.yaml + + - 200 + +.. rest_status_code:: error status.yaml + + - 400 + - 401 + - 404 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - backup_id: backup_id_request_path + +Response parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - id: backup_id_response + - share_id: backup_share_id + - status: backup_status + - size: backup_size + - availability_zone: backup_az + - name: name + - description: description + - created_at: created_at + - updated_at: updated_at + - progress: backup_progress + - restore_progress: backup_restore_progress + +Response example +---------------- + +.. literalinclude:: samples/share-backup-show-response.json + :language: javascript + + +Create share backup +~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: POST /v2/share-backups + +.. versionadded:: 2.80 + +Creates a backup from a share. + + +Response codes +-------------- + +.. rest_status_code:: success status.yaml + + - 202 + +.. rest_status_code:: error status.yaml + + - 400 + - 401 + - 403 + - 404 + - 409 + - 422 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - share_id: backup_share_id + - name: name_request + - description: description_request + - backup_options: backup_options_request + +Request example +--------------- + +.. literalinclude:: samples/share-backup-create-request.json + :language: javascript + +Response parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - id: backup_id_response + - share_id: backup_share_id + - status: backup_status + - size: backup_size + - availability_zone: backup_az + - name: name + - description: description + - created_at: created_at + - updated_at: updated_at + - progress: backup_progress + - restore_progress: backup_restore_progress + +Response example +---------------- + +.. literalinclude:: samples/share-backup-create-response.json + :language: javascript + + +Update share backup +~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: PUT /v2/share-backups/{backup_id} + +.. versionadded:: 2.80 + +Updates a share backup. + +You can update these attributes: + +- ``display_name``, which changes the ``name`` of the share backup. + +- ``display_description``, which changes the ``description`` of + the share backup. + +If you try to update other attributes, they retain their previous +values. + +Response codes +-------------- + +.. rest_status_code:: success status.yaml + + - 200 + +.. rest_status_code:: error status.yaml + + - 400 + - 401 + - 403 + - 404 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - backup_id: backup_id_request_path + - display_name: display_name_request + - display_description: display_description_request + +Request example +--------------- + +.. literalinclude:: samples/share-backup-update-request.json + :language: javascript + +Response parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - id: backup_id_response + - share_id: backup_share_id + - status: backup_status + - size: backup_size + - availability_zone: backup_az + - name: name + - description: description + - created_at: created_at + - updated_at: updated_at + - progress: backup_progress + - restore_progress: backup_restore_progress + +Response example +---------------- + +.. literalinclude:: samples/share-backup-update-response.json + :language: javascript + + +Delete share backup +~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: DELETE /v2/share-backups/{backup_id} + +.. versionadded:: 2.80 + +Deletes a share backup. + +Preconditions + +- Share backup status must be ``available`` or ``error``. + +Response codes +-------------- + +.. rest_status_code:: success status.yaml + + - 202 + +.. rest_status_code:: error status.yaml + + - 400 + - 401 + - 403 + - 404 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - backup_id: backup_id_request_path + + +Restore a share backup +~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: POST /v2/share-backups/{backup_id}/action + +.. versionadded:: 2.80 + +Restores a share backup into original share. + +Preconditions + +- Share backup status must be ``available``. + +Response codes +-------------- + +.. rest_status_code:: success status.yaml + + - 202 + +.. rest_status_code:: error status.yaml + + - 400 + - 401 + - 403 + - 404 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - backup_id: backup_id_request_path + +Request example +--------------- + +.. literalinclude:: samples/share-backup-restore-request.json + :language: javascript + +Response parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - backup_id: backup_id_response + - share_id: backup_share_id + +Response example +---------------- + +.. literalinclude:: samples/share-backup-restore-response.json + :language: javascript + + +Reset share backup status +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: POST /v2/share-backups/{backup_id}/action + +.. versionadded:: 2.80 + +Administrator only. Explicitly updates the state of a share backup. + +Use the ``policy.yaml`` file to grant permissions for this action +to other roles. + +Response codes +-------------- + +.. rest_status_code:: success status.yaml + + - 202 + +.. rest_status_code:: error status.yaml + + - 400 + - 401 + - 403 + - 404 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - project_id: project_id_path + - backup_id: backup_id_request_path + - status: backup_status_request + +Request example +--------------- + +.. literalinclude:: samples/share-backup-reset-status-request.json + :language: javascript diff --git a/api-ref/source/shares.inc b/api-ref/source/shares.inc index 827a0dc8e4..a82130754d 100644 --- a/api-ref/source/shares.inc +++ b/api-ref/source/shares.inc @@ -36,6 +36,12 @@ A share has one of these status values: +----------------------------------------+--------------------------------------------------------+ | Status | Description | +----------------------------------------+--------------------------------------------------------+ +| ``backup_creating`` | The share is being backed up. | ++----------------------------------------+--------------------------------------------------------+ +| ``backup_restoring`` | The share is being restored from backup. | ++----------------------------------------+--------------------------------------------------------+ +| ``backup_restoring_error`` | An error occurred during share backup restore. | ++----------------------------------------+--------------------------------------------------------+ | ``creating`` | The share is being created. | +----------------------------------------+--------------------------------------------------------+ | ``creating_from_snapshot`` | The share is being created from a parent snapshot. | @@ -122,6 +128,7 @@ Request - extra_specs: extra_specs_query - share_type_id: share_type_id_query - snapshot_id: snapshot_id_query + - source_backup_id: source_backup_id_query - host: host_query - share_network_id: share_network_id_query - project_id: project_id_query @@ -191,6 +198,7 @@ Request - share_type_id: share_type_id_query - name: name_query - snapshot_id: snapshot_id_query + - source_backup_id: source_backup_id_query - host: host_query - share_network_id: share_network_id_query - project_id: project_id_query @@ -221,6 +229,7 @@ Response parameters - description: description - project_id: project_id - snapshot_id: snapshot_id_shares_response + - source_backup_id: source_backup_id_shares_response - share_network_id: share_network_id - share_proto: share_proto - metadata: metadata diff --git a/devstack/plugin.sh b/devstack/plugin.sh index fd88ab58cf..761bf288e8 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -234,6 +234,14 @@ function configure_manila { iniset $MANILA_CONF DEFAULT server_migration_driver_continue_update_interval $MANILA_SERVER_MIGRATION_PERIOD_TASK_INTERVAL fi + if ! [[ -z $MANILA_CREATE_BACKUP_CONTINUE_TASK_INTERVAL ]]; then + iniset $MANILA_CONF DEFAULT driver_backup_continue_update_interval $MANILA_CREATE_BACKUP_CONTINUE_TASK_INTERVAL + fi + + if ! [[ -z $MANILA_RESTORE_BACKUP_CONTINUE_TASK_INTERVAL ]]; then + iniset $MANILA_CONF DEFAULT driver_restore_continue_update_interval $MANILA_RESTORE_BACKUP_CONTINUE_TASK_INTERVAL + fi + if ! [[ -z $MANILA_DATA_COPY_CHECK_HASH ]]; then iniset $MANILA_CONF DEFAULT check_hash $MANILA_DATA_COPY_CHECK_HASH fi diff --git a/manila/api/v1/share_unmanage.py b/manila/api/v1/share_unmanage.py index d77f4a017e..808e0db37a 100644 --- a/manila/api/v1/share_unmanage.py +++ b/manila/api/v1/share_unmanage.py @@ -64,6 +64,13 @@ class ShareUnmanageMixin(object): "'%(amount)s' dependent snapshot(s).") % { 's_id': id, 'amount': len(snapshots)} raise exc.HTTPForbidden(explanation=msg) + filters = {'share_id': id} + backups = self.share_api.db.share_backups_get_all(context, filters) + if backups: + msg = _("Share '%(s_id)s' can not be unmanaged because it has " + "'%(amount)s' dependent backup(s).") % { + 's_id': id, 'amount': len(backups)} + raise exc.HTTPForbidden(explanation=msg) self.share_api.unmanage(context, share) except exception.NotFound as e: raise exc.HTTPNotFound(explanation=e.msg) diff --git a/manila/api/v2/share_backups.py b/manila/api/v2/share_backups.py index 5867186519..d36d9e7672 100644 --- a/manila/api/v2/share_backups.py +++ b/manila/api/v2/share_backups.py @@ -21,6 +21,7 @@ from manila.api.views import share_backups as backup_view from manila import db from manila import exception from manila.i18n import _ +from manila import policy from manila import share @@ -72,6 +73,31 @@ class ShareBackupController(wsgi.Controller, wsgi.AdminActionsMixin): if sort_key == key: sort_key = key_dict[key] + if 'name' in search_opts: + search_opts['display_name'] = search_opts.pop('name') + if 'description' in search_opts: + search_opts['display_description'] = search_opts.pop( + 'description') + + # like filter + for key, db_key in (('name~', 'display_name~'), + ('description~', 'display_description~')): + if key in search_opts: + search_opts[db_key] = search_opts.pop(key) + + common.remove_invalid_options(context, search_opts, + self._get_backups_search_options()) + + # Read and remove key 'all_tenants' if was provided + search_opts['project_id'] = context.project_id + all_tenants = search_opts.pop('all_tenants', + search_opts.pop('all_projects', None)) + if all_tenants: + allowed_to_list_all_tenants = policy.check_policy( + context, 'share_backup', 'get_all_project', do_raise=False) + if allowed_to_list_all_tenants: + search_opts.pop('project_id') + share_id = req.params.get('share_id') if share_id: try: @@ -94,6 +120,11 @@ class ShareBackupController(wsgi.Controller, wsgi.AdminActionsMixin): return backups + def _get_backups_search_options(self): + """Return share backup search options allowed by non-admin.""" + return ('display_name', 'status', 'share_id', 'topic', 'display_name~', + 'display_description~', 'display_description') + @wsgi.Controller.api_version(MIN_SUPPORTED_API_VERSION, experimental=True) @wsgi.Controller.authorize('get') def show(self, req, id): @@ -190,7 +221,7 @@ class ShareBackupController(wsgi.Controller, wsgi.AdminActionsMixin): @wsgi.Controller.api_version(MIN_SUPPORTED_API_VERSION, experimental=True) @wsgi.Controller.authorize - @wsgi.response(202) + @wsgi.response(200) def update(self, req, id, body): """Update a backup.""" context = req.environ['manila.context'] @@ -217,6 +248,11 @@ class ShareBackupController(wsgi.Controller, wsgi.AdminActionsMixin): update_dict) return self._view_builder.detail(req, backup) + @wsgi.Controller.api_version(MIN_SUPPORTED_API_VERSION, experimental=True) + @wsgi.action('reset_status') + def backup_reset_status(self, req, id, body): + return self._reset_status(req, id, body) + def create_resource(): return wsgi.Resource(ShareBackupController()) diff --git a/manila/api/views/share_backups.py b/manila/api/views/share_backups.py index e8f860c398..12e5d5f629 100644 --- a/manila/api/views/share_backups.py +++ b/manila/api/views/share_backups.py @@ -36,8 +36,9 @@ class BackupViewBuilder(common.ViewBuilder): backup_dict = { 'id': backup.get('id'), + 'name': backup.get('display_name'), 'share_id': backup.get('share_id'), - 'backup_state': backup.get('status'), + 'status': backup.get('status'), } return {'share_backup': backup_dict} @@ -55,14 +56,16 @@ class BackupViewBuilder(common.ViewBuilder): context = request.environ['manila.context'] backup_dict = { 'id': backup.get('id'), - 'size': backup.get('size'), + 'name': backup.get('display_name'), 'share_id': backup.get('share_id'), - 'availability_zone': backup.get('availability_zone'), + 'status': backup.get('status'), + 'description': backup.get('display_description'), + 'size': backup.get('size'), 'created_at': backup.get('created_at'), 'updated_at': backup.get('updated_at'), - 'backup_state': backup.get('status'), - 'name': backup.get('display_name'), - 'description': backup.get('display_description'), + 'availability_zone': backup.get('availability_zone'), + 'progress': backup.get('progress'), + 'restore_progress': backup.get('restore_progress'), } if policy.check_is_host_admin(context): diff --git a/manila/api/views/shares.py b/manila/api/views/shares.py index 30c399ad7c..6ee2fe6157 100644 --- a/manila/api/views/shares.py +++ b/manila/api/views/shares.py @@ -38,6 +38,7 @@ class ViewBuilder(common.ViewBuilder): "add_progress_field", "translate_creating_from_snapshot_status", "add_share_recycle_bin_field", + "add_source_backup_id_field", ] def summary_list(self, request, shares, count=None): @@ -205,3 +206,7 @@ class ViewBuilder(common.ViewBuilder): share_dict['is_soft_deleted'] = share.get('is_soft_deleted') share_dict['scheduled_to_be_deleted_at'] = share.get( 'scheduled_to_be_deleted_at') + + @common.ViewBuilder.versioned_method("2.80") + def add_source_backup_id_field(self, context, share_dict, share): + share_dict['source_backup_id'] = share.get('source_backup_id') diff --git a/manila/data/manager.py b/manila/data/manager.py index 8d56441e4b..20312fff15 100644 --- a/manila/data/manager.py +++ b/manila/data/manager.py @@ -394,6 +394,8 @@ class DataManager(manager.Manager): share = self.db.share_get(context, share_id) backup = self.db.share_backup_get(context, backup_id) + self.db.share_backup_update(context, backup_id, {'host': self.host}) + LOG.info('Create backup started, backup: %(backup_id)s ' 'share: %(share_id)s.', {'backup_id': backup_id, 'share_id': share_id}) @@ -410,13 +412,21 @@ class DataManager(manager.Manager): self.db.share_backup_update( context, backup_id, {'status': constants.STATUS_ERROR, 'fail_reason': err}) + self.db.share_update( + context, share_id, {'status': constants.STATUS_AVAILABLE}) + self.db.share_backup_update( + context, backup_id, + {'status': constants.STATUS_AVAILABLE, 'progress': '100'}) + LOG.info("Created share backup %s successfully.", backup_id) @periodic_task.periodic_task( spacing=CONF.backup_continue_update_interval) def create_backup_continue(self, context): - filters = {'status': constants.STATUS_CREATING, - 'host': self.host, - 'topic': CONF.data_topic} + filters = { + 'status': constants.STATUS_CREATING, + 'host': self.host, + 'topic': CONF.data_topic + } backups = self.db.share_backups_get_all(context, filters) for backup in backups: @@ -426,17 +436,16 @@ class DataManager(manager.Manager): try: result = self.data_copy_get_progress(context, share_id) progress = result.get('total_progress', '0') - self.db.share_backup_update(context, backup_id, - {'progress': progress}) + backup_values = {'progress': progress} if progress == '100': self.db.share_update( context, share_id, {'status': constants.STATUS_AVAILABLE}) - self.db.share_backup_update( - context, backup_id, + backup_values.update( {'status': constants.STATUS_AVAILABLE}) LOG.info("Created share backup %s successfully.", backup_id) + self.db.share_backup_update(context, backup_id, backup_values) except Exception: LOG.warning("Failed to get progress of share %(share)s " "backing up in share_backup %(backup).", @@ -624,19 +633,29 @@ class DataManager(manager.Manager): {'status': constants.STATUS_BACKUP_RESTORING_ERROR}) self.db.share_backup_update( context, backup_id, - {'status': constants.STATUS_ERROR}) + {'status': constants.STATUS_AVAILABLE}) + self.db.share_update( + context, share_id, {'status': constants.STATUS_AVAILABLE}) + self.db.share_backup_update( + context, backup_id, + {'status': constants.STATUS_AVAILABLE, 'restore_progress': '100'}) + LOG.info("Share backup %s restored successfully.", backup_id) @periodic_task.periodic_task( spacing=CONF.restore_continue_update_interval) def restore_backup_continue(self, context): - filters = {'status': constants.STATUS_RESTORING, - 'host': self.host, - 'topic': CONF.data_topic} + filters = { + 'status': constants.STATUS_RESTORING, + 'host': self.host, + 'topic': CONF.data_topic + } backups = self.db.share_backups_get_all(context, filters) for backup in backups: backup_id = backup['id'] try: - filters = {'source_backup_id': backup_id} + filters = { + 'source_backup_id': backup_id, + } shares = self.db.share_get_all(context, filters) except Exception: LOG.warning('Failed to get shares for backup %s', backup_id) @@ -651,21 +670,21 @@ class DataManager(manager.Manager): try: result = self.data_copy_get_progress(context, share_id) progress = result.get('total_progress', '0') - self.db.share_backup_update(context, backup_id, - {'restore_progress': progress}) + backup_values = {'restore_progress': progress} if progress == '100': self.db.share_update( context, share_id, {'status': constants.STATUS_AVAILABLE}) - self.db.share_backup_update( - context, backup_id, + backup_values.update( {'status': constants.STATUS_AVAILABLE}) LOG.info("Share backup %s restored successfully.", backup_id) + self.db.share_backup_update(context, backup_id, + backup_values) except Exception: - LOG.warning("Failed to get progress of share_backup " - "%(backup)s restoring in share %(share).", - {'share': share_id, 'backup': backup_id}) + LOG.exception("Failed to get progress of share_backup " + "%(backup)s restoring in share %(share).", + {'share': share_id, 'backup': backup_id}) self.db.share_update( context, share_id, {'status': constants.STATUS_BACKUP_RESTORING_ERROR}) diff --git a/manila/db/migrations/alembic/versions/2d708a9a3ba9_backup_change_az_to_az_id.py b/manila/db/migrations/alembic/versions/2d708a9a3ba9_backup_change_az_to_az_id.py new file mode 100644 index 0000000000..ebd6bae21b --- /dev/null +++ b/manila/db/migrations/alembic/versions/2d708a9a3ba9_backup_change_az_to_az_id.py @@ -0,0 +1,94 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""backup_change_availability_zone_to_availability_zone_id + +Revision ID: 2d708a9a3ba9 +Revises: cb20f743ca7b +Create Date: 2023-08-24 11:01:41.134456 + +""" + +# revision identifiers, used by Alembic. +revision = '2d708a9a3ba9' +down_revision = 'cb20f743ca7b' + +from alembic import op +from sqlalchemy import Column, ForeignKey, String + +from manila.db.migrations import utils + + +def collect_existing_az(az_table, connection): + az_name_to_id_mapping = dict() + for az in connection.execute(az_table.select()): + if az.name in az_name_to_id_mapping: + continue + + az_name_to_id_mapping[az.name] = az.id + return az_name_to_id_mapping + + +def upgrade(): + connection = op.get_bind() + + op.add_column( + 'share_backups', + Column('availability_zone_id', String(36), + ForeignKey('availability_zones.id', name='sb_az_id_fk')) + ) + + # Collect existing AZs from availability_zones table + availability_zones_table = utils.load_table( + 'availability_zones', connection) + az_name_to_id_mapping = collect_existing_az( + availability_zones_table, connection,) + + # Map string AZ names to ID's in target table + # pylint: disable=no-value-for-parameter + set_az_id_in_table = lambda table, id, name: ( # noqa: E731 + op.execute( + table.update().where(table.c.availability_zone == name).values( + {'availability_zone_id': id}) + ) + ) + + share_backups_table = utils.load_table('share_backups', connection) + for name, id in az_name_to_id_mapping.items(): + set_az_id_in_table(share_backups_table, id, name) + + # Remove old AZ columns from table + op.drop_column('share_backups', 'availability_zone') + + +def downgrade(): + connection = op.get_bind() + + # Create old AZ fields + op.add_column('share_backups', + Column('availability_zone', String(length=255))) + + # Migrate data + az_table = utils.load_table('availability_zones', connection) + share_backups_table = utils.load_table('share_backups', connection) + + for az in connection.execute(az_table.select()): + # pylint: disable=no-value-for-parameter + op.execute( + share_backups_table.update().where( + share_backups_table.c.availability_zone_id == az.id + ).values({'availability_zone': az.name}) + ) + + # Remove AZ_id columns and AZ table + op.drop_constraint('sb_az_id_fk', 'share_backups', type_='foreignkey') + op.drop_column('share_backups', 'availability_zone_id') diff --git a/manila/db/sqlalchemy/api.py b/manila/db/sqlalchemy/api.py index 54f9e94435..2163eecd58 100644 --- a/manila/db/sqlalchemy/api.py +++ b/manila/db/sqlalchemy/api.py @@ -7071,6 +7071,7 @@ def _share_backup_create(context, share_id, values): if not values.get('id'): values['id'] = uuidutils.generate_uuid() values.update({'share_id': share_id}) + _ensure_availability_zone_exists(context, values) share_backup_ref = models.ShareBackup() share_backup_ref.update(values) @@ -7170,6 +7171,7 @@ def _backup_data_get_for_project(context, project_id, user_id): @oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True) @context_manager.writer def share_backup_update(context, backup_id, values): + _ensure_availability_zone_exists(context, values, strict=False) backup_ref = share_backup_get(context, backup_id) backup_ref.update(values) backup_ref.save(session=context.session) diff --git a/manila/db/sqlalchemy/models.py b/manila/db/sqlalchemy/models.py index 70cf1cba0c..d7ee175fa2 100644 --- a/manila/db/sqlalchemy/models.py +++ b/manila/db/sqlalchemy/models.py @@ -1504,6 +1504,11 @@ class ShareBackup(BASE, ManilaBase): def name(self): return CONF.share_backup_name_template % self.id + @property + def availability_zone(self): + if self._availability_zone: + return self._availability_zone['name'] + deleted = Column(String(36), default='False') user_id = Column(String(255), nullable=False) project_id = Column(String(255), nullable=False) @@ -1512,13 +1517,26 @@ class ShareBackup(BASE, ManilaBase): size = Column(Integer) host = Column(String(255)) topic = Column(String(255)) - availability_zone = Column(String(255)) display_name = Column(String(255)) display_description = Column(String(255)) progress = Column(String(32)) restore_progress = Column(String(32)) status = Column(String(255)) fail_reason = Column(String(1023)) + availability_zone_id = Column(String(36), + ForeignKey('availability_zones.id'), + nullable=True) + + _availability_zone = orm.relationship( + "AvailabilityZone", + lazy='immediate', + primaryjoin=( + 'and_(' + 'ShareBackup.availability_zone_id == ' + 'AvailabilityZone.id, ' + 'AvailabilityZone.deleted == \'False\')' + ) + ) def register_models(): diff --git a/manila/policies/share_backup.py b/manila/policies/share_backup.py index f2d8efcbc7..5f87661650 100644 --- a/manila/policies/share_backup.py +++ b/manila/policies/share_backup.py @@ -39,6 +39,12 @@ deprecated_backup_get_all = policy.DeprecatedRule( deprecated_reason=DEPRECATED_REASON, deprecated_since='2023.2/Bobcat', ) +deprecated_get_all_project = policy.DeprecatedRule( + name=BASE_POLICY_NAME % 'get_all_project', + check_str=base.RULE_ADMIN_API, + deprecated_reason=DEPRECATED_REASON, + deprecated_since='2023.2/Bobcat', +) deprecated_backup_restore = policy.DeprecatedRule( name=BASE_POLICY_NAME % 'restore', check_str=base.RULE_ADMIN_OR_OWNER, @@ -57,6 +63,12 @@ deprecated_backup_delete = policy.DeprecatedRule( deprecated_reason=DEPRECATED_REASON, deprecated_since='2023.2/Bobcat', ) +deprecated_backup_reset_status = policy.DeprecatedRule( + name=BASE_POLICY_NAME % 'reset_status', + check_str=base.RULE_ADMIN_API, + deprecated_reason=DEPRECATED_REASON, + deprecated_since='2023.2/Bobcat', +) share_backup_policies = [ @@ -107,6 +119,24 @@ share_backup_policies = [ ], deprecated_rule=deprecated_backup_get_all, ), + policy.DocumentedRuleDefault( + name=BASE_POLICY_NAME % 'get_all_project', + check_str=base.ADMIN, + scope_types=['project'], + description="Get share backups of all projects.", + operations=[ + { + 'method': 'GET', + 'path': '/share-backups?all_tenants=1' + }, + { + 'method': 'GET', + 'path': '/share-backups/detail?all_tenants=1' + } + ], + deprecated_rule=deprecated_get_all_project + ), + policy.DocumentedRuleDefault( name=BASE_POLICY_NAME % 'restore', check_str=base.ADMIN_OR_PROJECT_MEMBER, @@ -120,6 +150,19 @@ share_backup_policies = [ ], deprecated_rule=deprecated_backup_restore, ), + policy.DocumentedRuleDefault( + name=BASE_POLICY_NAME % 'reset_status', + check_str=base.ADMIN, + scope_types=['project'], + description="Reset status.", + operations=[ + { + 'method': 'POST', + 'path': '/share-backups/{backup_id}/action', + } + ], + deprecated_rule=deprecated_backup_reset_status + ), policy.DocumentedRuleDefault( name=BASE_POLICY_NAME % 'update', check_str=base.ADMIN_OR_PROJECT_MEMBER, diff --git a/manila/share/api.py b/manila/share/api.py index be00d27050..602e5ab140 100644 --- a/manila/share/api.py +++ b/manila/share/api.py @@ -3867,6 +3867,7 @@ class API(base.Base): raise exception.BackupLimitExceeded( allowed=quotas[over]) + backup_ref = {} try: backup_ref = self.db.share_backup_create( context, share['id'], @@ -3879,6 +3880,7 @@ class API(base.Base): 'display_description': backup.get('description'), 'display_name': backup.get('name'), 'size': share['size'], + 'availability_zone': share['instance']['availability_zone'] } ) QUOTAS.commit(context, reservations) @@ -3891,15 +3893,16 @@ class API(base.Base): {'status': constants.STATUS_BACKUP_CREATING}) backup_ref['backup_options'] = backup.get('backup_options', {}) + backup_values = {} if backup_ref['backup_options']: topic = CONF.share_topic + backup_ref['host'] = share_utils.extract_host(share['host']) + backup_values.update({'host': backup_ref['host']}) else: topic = CONF.data_topic - backup_ref['host'] = share['host'] - self.db.share_backup_update( - context, backup_ref['id'], - {'host': backup_ref['host'], 'topic': topic}) + backup_values.update({'topic': topic}) + self.db.share_backup_update(context, backup_ref['id'], backup_values) if topic == CONF.share_topic: self.share_rpcapi.create_backup(context, backup_ref) diff --git a/manila/share/manager.py b/manila/share/manager.py index 2bdccf22b4..0e28aeeb23 100644 --- a/manila/share/manager.py +++ b/manila/share/manager.py @@ -5217,7 +5217,9 @@ class ShareManager(manager.SchedulerDependentManager): for backup in backups: backup_id = backup['id'] try: - filters = {'source_backup_id': backup_id} + filters = { + 'source_backup_id': backup_id, + } shares = self.db.share_get_all(context, filters) except Exception: LOG.warning('Failed to get shares for backup %s', backup_id) @@ -5247,9 +5249,9 @@ class ShareManager(manager.SchedulerDependentManager): LOG.info("Share backup %s restored successfully.", backup_id) except Exception: - LOG.warning("Failed to get progress of share_backup " - "%(backup)s restoring in share %(share).", - {'share': share_id, 'backup': backup_id}) + LOG.exception("Failed to get progress of share_backup " + "%(backup)s restoring in share %(share).", + {'share': share_id, 'backup': backup_id}) self.db.share_update( context, share_id, {'status': constants.STATUS_BACKUP_RESTORING_ERROR}) diff --git a/manila/tests/api/v1/test_share_unmanage.py b/manila/tests/api/v1/test_share_unmanage.py index 61cd8c566a..8227b14c3e 100644 --- a/manila/tests/api/v1/test_share_unmanage.py +++ b/manila/tests/api/v1/test_share_unmanage.py @@ -60,6 +60,9 @@ class ShareUnmanageTest(test.TestCase): self.mock_object( self.controller.share_api.db, 'share_snapshot_get_all_for_share', mock.Mock(return_value=[])) + self.mock_object( + self.controller.share_api.db, 'share_backups_get_all', + mock.Mock(return_value=[])) actual_result = self.controller.unmanage(self.request, share['id']) @@ -67,6 +70,10 @@ class ShareUnmanageTest(test.TestCase): (self.controller.share_api.db.share_snapshot_get_all_for_share. assert_called_once_with( self.request.environ['manila.context'], share['id'])) + filters = {'share_id': 'foo_id'} + (self.controller.share_api.db.share_backups_get_all. + assert_called_once_with( + self.request.environ['manila.context'], filters)) self.controller.share_api.get.assert_called_once_with( self.request.environ['manila.context'], share['id']) share_api.API.unmanage.assert_called_once_with( @@ -99,6 +106,32 @@ class ShareUnmanageTest(test.TestCase): self.mock_policy_check.assert_called_once_with( self.context, self.resource_name, 'unmanage') + def test_unmanage_share_that_has_backups(self): + share = dict(status=constants.STATUS_AVAILABLE, id='foo_id', + instance={}) + backups = ['foo', 'bar'] + self.mock_object(self.controller.share_api, 'unmanage') + self.mock_object( + self.controller.share_api.db, 'share_backups_get_all', + mock.Mock(return_value=backups)) + self.mock_object( + self.controller.share_api, 'get', + mock.Mock(return_value=share)) + + self.assertRaises( + webob.exc.HTTPForbidden, + self.controller.unmanage, self.request, share['id']) + + self.assertFalse(self.controller.share_api.unmanage.called) + filters = {'share_id': 'foo_id'} + (self.controller.share_api.db.share_backups_get_all. + assert_called_once_with( + self.request.environ['manila.context'], filters)) + self.controller.share_api.get.assert_called_once_with( + self.request.environ['manila.context'], share['id']) + self.mock_policy_check.assert_called_once_with( + self.context, self.resource_name, 'unmanage') + def test_unmanage_share_that_has_replicas(self): share = dict(status=constants.STATUS_AVAILABLE, id='foo_id', instance={}, has_replicas=True) @@ -106,6 +139,8 @@ class ShareUnmanageTest(test.TestCase): 'unmanage') mock_db_snapshots_get = self.mock_object( self.controller.share_api.db, 'share_snapshot_get_all_for_share') + mock_db_backups_get = self.mock_object( + self.controller.share_api.db, 'share_backups_get_all') self.mock_object( self.controller.share_api, 'get', mock.Mock(return_value=share)) @@ -116,6 +151,7 @@ class ShareUnmanageTest(test.TestCase): self.assertFalse(mock_api_unmanage.called) self.assertFalse(mock_db_snapshots_get.called) + self.assertFalse(mock_db_backups_get.called) self.controller.share_api.get.assert_called_once_with( self.request.environ['manila.context'], share['id']) self.mock_policy_check.assert_called_once_with( diff --git a/manila/tests/api/v2/test_share_backups.py b/manila/tests/api/v2/test_share_backups.py index 19dfee91db..6e2bfefeb9 100644 --- a/manila/tests/api/v2/test_share_backups.py +++ b/manila/tests/api/v2/test_share_backups.py @@ -68,21 +68,23 @@ class ShareBackupsApiTest(test.TestCase): def _get_fake_backup(self, admin=False, summary=False, **values): backup = fake_share.fake_backup(**values) backup['updated_at'] = '2016-06-12T19:57:56.506805' - expected_keys = {'id', 'share_id', 'backup_state'} + expected_keys = {'id', 'share_id', 'status'} expected_backup = {key: backup[key] for key in backup if key in expected_keys} + expected_backup.update({'name': backup.get('display_name')}) if not summary: expected_backup.update({ 'id': backup.get('id'), - 'size': backup.get('size'), 'share_id': backup.get('share_id'), - 'availability_zone': backup.get('availability_zone'), - 'created_at': backup.get('created_at'), - 'backup_state': backup.get('status'), - 'updated_at': backup.get('updated_at'), - 'name': backup.get('display_name'), + 'status': backup.get('status'), 'description': backup.get('display_description'), + 'size': backup.get('size'), + 'created_at': backup.get('created_at'), + 'updated_at': backup.get('updated_at'), + 'availability_zone': backup.get('availability_zone'), + 'progress': backup.get('progress'), + 'restore_progress': backup.get('restore_progress'), }) if admin: expected_backup.update({ @@ -102,7 +104,7 @@ class ShareBackupsApiTest(test.TestCase): self.mock_policy_check.assert_called_once_with( self.member_context, self.resource_name, 'get_all') - def test_list_share_backups_summary(self): + def test_list_backups_summary_with_share_id(self): fake_backup, expected_backup = self._get_fake_backup(summary=True) self.mock_object(share.API, 'get', mock.Mock(return_value={'id': 'FAKE_SHAREID'})) @@ -379,10 +381,10 @@ class ShareBackupsApiTest(test.TestCase): def test_delete_exception(self): fake_backup_1 = self._get_fake_backup( share_id='FAKE_SHARE_ID', - backup_state=constants.STATUS_BACKUP_CREATING)[0] + status=constants.STATUS_BACKUP_CREATING)[0] fake_backup_2 = self._get_fake_backup( share_id='FAKE_SHARE_ID', - backup_state=constants.STATUS_BACKUP_CREATING)[0] + status=constants.STATUS_BACKUP_CREATING)[0] exception_type = exception.InvalidBackup(reason='xyz') self.mock_object(share_backups.db, 'share_backup_get', mock.Mock(return_value=fake_backup_1)) @@ -398,7 +400,7 @@ class ShareBackupsApiTest(test.TestCase): def test_delete(self): fake_backup = self._get_fake_backup( share_id='FAKE_SHARE_ID', - backup_state=constants.STATUS_AVAILABLE)[0] + status=constants.STATUS_AVAILABLE)[0] self.mock_object(share_backups.db, 'share_backup_get', mock.Mock(return_value=fake_backup)) self.mock_object(share.API, 'delete_share_backup') @@ -424,7 +426,7 @@ class ShareBackupsApiTest(test.TestCase): body = {'restore': {'share_id': 'fake_id'}} fake_backup = self._get_fake_backup( share_id='FAKE_SHARE_ID', - backup_state=constants.STATUS_AVAILABLE)[0] + status=constants.STATUS_AVAILABLE)[0] self.mock_object(share_backups.db, 'share_backup_get', mock.Mock(return_value=fake_backup)) @@ -447,7 +449,7 @@ class ShareBackupsApiTest(test.TestCase): def test_update(self): fake_backup = self._get_fake_backup( share_id='FAKE_SHARE_ID', - backup_state=constants.STATUS_AVAILABLE)[0] + status=constants.STATUS_AVAILABLE)[0] self.mock_object(share_backups.db, 'share_backup_get', mock.Mock(return_value=fake_backup)) diff --git a/manila/tests/api/v2/test_shares.py b/manila/tests/api/v2/test_shares.py index 5c4e12bffb..2fbd8caab9 100644 --- a/manila/tests/api/v2/test_shares.py +++ b/manila/tests/api/v2/test_shares.py @@ -3152,6 +3152,10 @@ class ShareManageTest(test.TestCase): api_version.APIVersionRequest('2.8')): share['is_public'] = data['share']['is_public'] + if (api_version.APIVersionRequest(version) >= + api_version.APIVersionRequest('2.80')): + share['source_backup_id'] = None + req = fakes.HTTPRequest.blank('/v2/fake/shares/manage', version=version, use_admin_context=True) diff --git a/manila/tests/data/test_manager.py b/manila/tests/data/test_manager.py index cd9b0bc388..59b4dd792d 100644 --- a/manila/tests/data/test_manager.py +++ b/manila/tests/data/test_manager.py @@ -522,12 +522,19 @@ class DataManagerTestCase(test.TestCase): # mocks self.mock_object(db, 'share_update') + self.mock_object(db, 'share_backup_update') self.mock_object(db, 'share_get', mock.Mock(return_value=share_info)) self.mock_object(db, 'share_backup_get', mock.Mock(return_value=backup_info)) self.mock_object(self.manager, '_run_backup', mock.Mock(side_effect=None)) self.manager.create_backup(self.context, backup_info) + db.share_update.assert_called_with( + self.context, share_info['id'], + {'status': constants.STATUS_AVAILABLE}) + db.share_backup_update.assert_called_with( + self.context, backup_info['id'], + {'status': constants.STATUS_AVAILABLE, 'progress': '100'}) def test_create_share_backup_exception(self): share_info = db_utils.create_share(status=constants.STATUS_AVAILABLE) @@ -550,7 +557,7 @@ class DataManagerTestCase(test.TestCase): db.share_update.assert_called_with( self.context, share_info['id'], {'status': constants.STATUS_AVAILABLE}) - db.share_backup_update.assert_called_once() + db.share_backup_update.assert_called() @ddt.data('90', '100') def test_create_share_backup_continue(self, progress): @@ -571,7 +578,7 @@ class DataManagerTestCase(test.TestCase): if progress == '100': db.share_backup_update.assert_called_with( self.context, backup_info['id'], - {'status': constants.STATUS_AVAILABLE}) + {'status': constants.STATUS_AVAILABLE, 'progress': '100'}) db.share_update.assert_called_with( self.context, share_info['id'], {'status': constants.STATUS_AVAILABLE}) @@ -678,11 +685,18 @@ class DataManagerTestCase(test.TestCase): # mocks self.mock_object(db, 'share_update') + self.mock_object(db, 'share_backup_update') self.mock_object(db, 'share_get', mock.Mock(return_value=share_info)) self.mock_object(db, 'share_backup_get', mock.Mock(return_value=backup_info)) self.mock_object(self.manager, '_run_restore') self.manager.restore_backup(self.context, backup_info, share_id) + db.share_update.assert_called_with( + self.context, share_info['id'], + {'status': constants.STATUS_AVAILABLE}) + db.share_backup_update.assert_called_with( + self.context, backup_info['id'], + {'status': constants.STATUS_AVAILABLE, 'restore_progress': '100'}) def test_restore_share_backup_exception(self): share_info = db_utils.create_share(status=constants.STATUS_AVAILABLE) @@ -730,7 +744,8 @@ class DataManagerTestCase(test.TestCase): if progress == '100': db.share_backup_update.assert_called_with( self.context, backup_info['id'], - {'status': constants.STATUS_AVAILABLE}) + {'status': constants.STATUS_AVAILABLE, + 'restore_progress': '100'}) db.share_update.assert_called_with( self.context, share_info['id'], {'status': constants.STATUS_AVAILABLE}) diff --git a/manila/tests/db/sqlalchemy/test_api.py b/manila/tests/db/sqlalchemy/test_api.py index 2f15730829..45518a06f8 100644 --- a/manila/tests/db/sqlalchemy/test_api.py +++ b/manila/tests/db/sqlalchemy/test_api.py @@ -5385,16 +5385,6 @@ class ShareBackupDatabaseAPITestCase(BaseDatabaseAPITestCase): self.ctxt, self.share_id, self.backup) self._check_fields(expected=self.backup, actual=result) - def test_create_with_duplicated_id(self): - db_api.share_backup_create( - self.ctxt, self.share_id, self.backup) - - self.assertRaises(db_exception.DBDuplicateEntry, - db_api.share_backup_create, - self.ctxt, - self.share_id, - self.backup) - def test_get(self): db_api.share_backup_create( self.ctxt, self.share_id, self.backup) diff --git a/manila/tests/fake_share.py b/manila/tests/fake_share.py index 953e0e91c1..78d5c52396 100644 --- a/manila/tests/fake_share.py +++ b/manila/tests/fake_share.py @@ -328,7 +328,6 @@ def fake_backup(as_primitive=True, **kwargs): 'user_id': 'fake', 'project_id': 'fake', 'availability_zone': 'fake_availability_zone', - 'backup_state': constants.STATUS_CREATING, 'status': constants.STATUS_CREATING, 'progress': '0', 'restore_progress': '0',