From 91837d4fa7325e909b5f5a655336015688edb47c Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Sat, 23 Jan 2021 23:45:33 +0100 Subject: [PATCH] Add new parameters to configure nova-compute direct rbd image download If rbd is used for glance, but compute is using local ephemeral storage, nova-compute can direct download the images in this scenario from the glance ceph pool via rbd, instead going through glance api. This change introduce new compute role based parameters to enable direct download of glance images via rbd. If NovaGlanceEnableRbdDownload is set, per default the global RBD glance parameters are used, CephClientUserName GlanceRbdPoolName and CephClusterName for the used ceph.conf. Glance also support multi storage backends which can be configured using GlanceMultistoreConfig. If additional RBD glance backends are configured, the NovaGlanceRbdDownloadMultistoreID can be used to pointing to the hash key (backend ID) of GlanceMultistoreConfig to use. Depends-On: https://review.opendev.org/c/openstack/puppet-tripleo/+/772168 Depends-On: https://review.opendev.org/c/openstack/puppet-nova/+/770687 Change-Id: I020da468d909bd98819f1e3618bf905260791d9b --- .../nova/nova-compute-container-puppet.yaml | 141 ++++++++++++++++-- ..._glance_rbd_download-e945933da26f10f0.yaml | 17 +++ 2 files changed, 146 insertions(+), 12 deletions(-) create mode 100644 releasenotes/notes/nova_direct_glance_rbd_download-e945933da26f10f0.yaml diff --git a/deployment/nova/nova-compute-container-puppet.yaml b/deployment/nova/nova-compute-container-puppet.yaml index 9ecdf94e72..ab84cf41c6 100644 --- a/deployment/nova/nova-compute-container-puppet.yaml +++ b/deployment/nova/nova-compute-container-puppet.yaml @@ -559,6 +559,51 @@ parameters: default: 0 tags: - role_specific + GlanceRbdPoolName: + default: images + type: string + GlanceMultistoreConfig: + type: json + default: {} + description: | + Dictionary of settings when configuring additional glance backends. The + hash key is the backend ID, and the value is a dictionary of parameter + values unique to that backend. Multiple rbd backends are allowed, but + cinder, file and swift backends are limited to one each. Example: + # Default glance store is rbd. + GlanceBackend: rbd + GlanceStoreDescription: 'Default rbd store' + # GlanceMultistoreConfig specifies a second rbd backend, plus a cinder + # backend. + GlanceMultistoreConfig: + rbd2_store: + GlanceBackend: rbd + GlanceStoreDescription: 'Second rbd store' + CephClusterName: ceph2 + # Override CephClientUserName if this cluster uses a different + # client name. + CephClientUserName: client2 + cinder_store: + GlanceBackend: cinder + GlanceStoreDescription: 'Cinder store' + NovaGlanceEnableRbdDownload: + type: boolean + description: > + Enable download of Glance images directly via RBD. + default: false + tags: + - role_specific + NovaGlanceRbdDownloadMultistoreID: + type: string + description: > + The hash key, which is the backend ID, of the GlanceMultistoreConfig to + be used for the role where NovaGlanceEnableRbdDownload is enabled and + defaults should be overridden. If CephClientUserName or GlanceRbdPoolName + are not set in the GlanceMultistoreConfig, the global values of those + parameters will be used. + default: '' + tags: + - role_specific # DEPRECATED: the following options are deprecated and are currently maintained # for backwards compatibility. They will be removed in future release. @@ -668,6 +713,8 @@ resources: nova::compute::image_type_exclude_list: NovaImageTypeExcludeList nova_permit_post_copy: NovaLiveMigrationPermitPostCopy nova_permit_auto_converge: NovaLiveMigrationPermitAutoConverge + nova_glance_enable_rbd_download: NovaGlanceEnableRbdDownload + nova_glance_rbd_download_multistore_id: NovaGlanceRbdDownloadMultistoreID - values: {get_param: [RoleParameters]} - values: NovaVcpuPinSet: {get_param: NovaVcpuPinSet} @@ -756,7 +803,8 @@ resources: NovaLiveMigrationPermitPostCopy: {get_param: NovaLiveMigrationPermitPostCopy} NovaLiveMigrationPermitAutoConverge: {get_param: NovaLiveMigrationPermitAutoConverge} NovaLibvirtMaxQueues: {get_param: NovaLibvirtMaxQueues} - + NovaGlanceEnableRbdDownload: {get_param: NovaGlanceEnableRbdDownload} + NovaGlanceRbdDownloadMultistoreID: {get_param: NovaGlanceRbdDownloadMultistoreID} conditions: enable_instance_ha: {equals: [{get_param: EnableInstanceHA}, true]} @@ -823,6 +871,19 @@ conditions: - equals: [{get_param: NovaLibvirtMaxQueues}, 0] - equals: [{get_param: [RoleParameters, NovaLibvirtMaxQueues]}, 0] + glance_enable_rbd_download_set: + or: + - and: + - equals: [{get_param: NovaGlanceEnableRbdDownload}, true] + - equals: [{get_param: [RoleParameters, NovaGlanceEnableRbdDownload]}, ''] + - equals: [{get_param: [RoleParameters, NovaGlanceEnableRbdDownload]}, true] + + nova_glance_rbd_download_multistore_id_set: + not: + and: + - equals: [{get_param: [RoleParameters, NovaGlanceRbdDownloadMultistoreID]}, ''] + - equals: [{get_param: NovaGlanceRbdDownloadMultistoreID}, ''] + outputs: role_data: description: Role data for the Nova Compute service. @@ -945,6 +1006,48 @@ outputs: - nova_libvirt_max_queues_set - nova::compute::libvirt::max_queues: {get_attr: [RoleParametersValue, value, nova_libvirt_max_queues]} - {} + - if: + - glance_enable_rbd_download_set + - + nova::glance::enable_rbd_download: {get_attr: [RoleParametersValue, value, nova_glance_enable_rbd_download]} + nova::glance::rbd_user: + if: + - nova_glance_rbd_download_multistore_id_set + - + yaql: + expression: let(k => $.data.key, d => $.data.default_ceph_client_user_name) -> $.data.multistore.get($k).get("CephClientUserName", $d) + data: + key: {get_attr: [RoleParametersValue, value, nova_glance_rbd_download_multistore_id]} + multistore: {get_param: GlanceMultistoreConfig} + default_ceph_client_user_name: {get_param: CephClientUserName} + - {get_param: CephClientUserName} + nova::glance::rbd_pool: + if: + - nova_glance_rbd_download_multistore_id_set + - + yaql: + expression: let(k => $.data.key, d => $.data.default_glance_rbd_pool_name) -> $.data.multistore.get($k).get("GlanceRbdPoolName", $d) + data: + key: {get_attr: [RoleParametersValue, value, nova_glance_rbd_download_multistore_id]} + multistore: {get_param: GlanceMultistoreConfig} + default_glance_rbd_pool_name: {get_param: GlanceRbdPoolName} + - {get_param: GlanceRbdPoolName} + nova::glance::rbd_ceph_conf: + list_join: + - '' + - - '/etc/ceph/' + - + if: + - nova_glance_rbd_download_multistore_id_set + - + yaql: + expression: let(k => $.data.key) -> list(coalesce($.data.multistore, {}).get($k)).select($.get("CephClusterName")) + data: + key: {get_attr: [RoleParametersValue, value, nova_glance_rbd_download_multistore_id]} + multistore: {get_param: GlanceMultistoreConfig} + - {get_param: CephClusterName} + - '.conf' + - {} service_config_settings: rsyslog: tripleo_logging_sources_nova_compute: @@ -992,17 +1095,31 @@ outputs: merge: true preserve_properties: true permissions: - - path: /var/log/nova - owner: nova:nova - recurse: true - - path: - str_replace: - template: /etc/ceph/CLUSTER.client.USER.keyring - params: - CLUSTER: {get_param: CephClusterName} - USER: {get_param: CephClientUserName} - owner: nova:nova - perm: '0600' + list_concat: + - + - path: /var/log/nova + owner: nova:nova + recurse: true + - path: + str_replace: + template: /etc/ceph/CLUSTER.client.USER.keyring + params: + CLUSTER: {get_param: CephClusterName} + USER: {get_param: CephClientUserName} + owner: nova:nova + perm: '0600' + - repeat: + template: + path: /etc/ceph/<%keyring%> + owner: nova:nova + perm: '0600' + for_each: + <%keyring%>: + yaql: + expression: let(u => $.data.user) -> $.data.multistore.values().where($.get("CephClusterName")).select("{0}.client.{1}.keyring".format($.CephClusterName, $.get("CephClientUserName", $u) )) + data: + user: {get_param: CephClientUserName} + multistore: {get_param: GlanceMultistoreConfig} container_config_scripts: map_merge: - {get_attr: [ContainersCommon, container_config_scripts]} diff --git a/releasenotes/notes/nova_direct_glance_rbd_download-e945933da26f10f0.yaml b/releasenotes/notes/nova_direct_glance_rbd_download-e945933da26f10f0.yaml new file mode 100644 index 0000000000..68836a0612 --- /dev/null +++ b/releasenotes/notes/nova_direct_glance_rbd_download-e945933da26f10f0.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + Introduce new compute role based parameter NovaGlanceEnableRbdDownload + to enable direct download if rbd is used for glance, but compute is using + local ephemeral storage, to allow nova-compute to direct download the + images in this scenario from the glance ceph pool via rbd, instead going + through glance api. + If NovaGlanceEnableRbdDownload is set, per default the global RBD glance + parameters are used, CephClientUserName GlanceRbdPoolName and + CephClusterName for the used ceph.conf. + Glance supports multi storage backends which can be configured using + GlanceMultistoreConfig. If additional RBD glance backends are configured, + the NovaGlanceRbdDownloadMultistoreID can be used to pointing to the + hash key (backend ID) of GlanceMultistoreConfig to use. If + CephClientUserName or GlanceRbdPoolName are not set in the + GlanceMultistoreConfig, the global values of those parameters will be used.