From 3b8e6f78e19e776c087dc5c3ff225703b5c487bc Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Thu, 26 Mar 2020 12:26:53 +0100 Subject: [PATCH] Use lists for storing host entries in Heat This simplifies all the split/join transformations and improves the memory footprint to a reduced list of unique entries for HostsEntryValue (originally required for storing the ultimate data for hosts entries in a form of a quite long single-line string value). That improves the hosts entries processing for large scale deployments and removes possible limitations to the sizes of strings. Closes-bug: #1869375 Change-Id: I5ac498621e9e3c49def565744a7b521cb2cc5c25 Signed-off-by: Bogdan Dobrelya --- common/deploy-steps.j2 | 6 ++--- overcloud.j2.yaml | 49 ++++++++++++++++++---------------------- puppet/role.role.j2.yaml | 40 ++++++++++++++++++-------------- 3 files changed, 48 insertions(+), 47 deletions(-) diff --git a/common/deploy-steps.j2 b/common/deploy-steps.j2 index 029a5e7744..e7afd21eb3 100644 --- a/common/deploy-steps.j2 +++ b/common/deploy-steps.j2 @@ -149,9 +149,9 @@ parameters: Currently supports tarballs and RPM packages. type: comma_delimited_list HostsEntry: - default: '' - type: string - description: A string of entries to be added to /etc/hosts on each node. + default: [] + type: comma_delimited_list + description: A list of entries to be added to /etc/hosts on each node. AnsibleHostVarsMap: type: json default: {} diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml index 45f7760244..401c01b110 100644 --- a/overcloud.j2.yaml +++ b/overcloud.j2.yaml @@ -399,11 +399,10 @@ resources: VipHosts: type: OS::Heat::Value properties: - type: string + type: json value: - list_join: - - "\n" - - - str_replace: + entries: + - str_replace: template: IP HOST params: IP: {get_attr: [VipMap, net_ip_map, ctlplane]} @@ -791,34 +790,31 @@ resources: for_each: hostname: {get_param: DeploymentServerBlacklist} + # Single value comma delimited lists may be stored in strings + # that fails lists concat. Make them always lists (nested are OK) HostsValue: type: OS::Heat::Value properties: + type: json value: - list_join: - - "\n" - - {get_param: UndercloudHostsEntries} - - - if: - - add_vips_to_etc_hosts - - {get_attr: [VipHosts, value]} - - '' - - + entries: + - - {get_param: UndercloudHostsEntries} + - - if: + - add_vips_to_etc_hosts + - {get_attr: [VipHosts, value, entries]} + - [] {%- for role in roles %} - - list_join: - - "" - - {get_attr: [{{role.name}}, hosts_entry]} + - {get_attr: [{{role.name}}, hosts_entry, entries]} {%- endfor %} - - {get_param: ExtraHostFileEntries} + - - {get_param: ExtraHostFileEntries} HostsEntryValue: type: OS::Heat::Value properties: value: - list_join: - - ' ' - - str_split: - - '\n' - - {get_attr: [HostsValue, value]} + yaql: + expression: coalesce($.data, []).where($ != null).where($ != []).flatten().distinct() + data: {get_attr: [HostsValue, value, entries]} CloudNames: type: OS::Heat::Value @@ -1130,8 +1126,8 @@ resources: VipHostsEntries: if: - add_vips_to_etc_hosts - - {get_attr: [VipHosts, value]} - - '' + - {get_attr: [VipHosts, value, entries]} + - [] KeystoneResourcesConfigs: map_merge: {% for role in roles %} @@ -1165,10 +1161,9 @@ outputs: hostname-based access to the deployed nodes (useful for testing without setting up a DNS). value: - list_join: - - "\n" - - - {get_attr: [HostsEntryValue, value]} - - - {get_attr: [VipHosts, value]} + list_concat_unique: + - {get_attr: [HostsEntryValue, value]} + - {get_attr: [VipHosts, value, entries]} EnabledServices: description: The services enabled on each role value: diff --git a/puppet/role.role.j2.yaml b/puppet/role.role.j2.yaml index 9b2774a4da..64fa4300c5 100644 --- a/puppet/role.role.j2.yaml +++ b/puppet/role.role.j2.yaml @@ -632,27 +632,33 @@ outputs: HOSTNAME: {get_attr:[{{server_resource_name}}, name]} hosts_entry: value: - str_replace: - template: | - PRIMARYIP PRIMARYHOST.DOMAIN PRIMARYHOST + entries: + - str_replace: + template: PRIMARYIP PRIMARYHOST.DOMAIN PRIMARYHOST + params: + PRIMARYIP: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, {{role.name}}HostnameResolveNetwork]}]} + DOMAIN: {get_param: CloudDomain} + PRIMARYHOST: {get_attr: [{{server_resource_name}}, name]} {%- for network in networks %} {%- if network.enabled|default(true) and network.name in role.networks|default([]) %} - {{network.name}}IP {{network.name}}HOST.DOMAIN {{network.name}}HOST + - str_replace: + template: {{network.name}}IP {{network.name}}HOST.DOMAIN {{network.name}}HOST + params: + DOMAIN: {get_param: CloudDomain} + {%- for network in networks %} + {%- if network.enabled|default(true) and network.name in role.networks|default([]) %} + {{network.name}}IP: {get_attr: [{{network.name}}Port, ip_address]} + {{network.name}}HOST: {get_attr: [NetHostMap, value, {{network.name_lower|default(network.name.lower())}}, short]} + {%- endif %} + {%- endfor %} {%- endif %} {%- endfor %} - CTLPLANEIP CTLPLANEHOST.DOMAIN CTLPLANEHOST - params: - PRIMARYIP: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, {{role.name}}HostnameResolveNetwork]}]} - DOMAIN: {get_param: CloudDomain} - PRIMARYHOST: {get_attr: [{{server_resource_name}}, name]} - {%- for network in networks %} - {%- if network.enabled|default(true) and network.name in role.networks|default([]) %} - {{network.name}}IP: {get_attr: [{{network.name}}Port, ip_address]} - {{network.name}}HOST: {get_attr: [NetHostMap, value, {{network.name_lower|default(network.name.lower())}}, short]} - {%- endif %} - {%- endfor %} - CTLPLANEIP: {get_attr: [{{server_resource_name}}, networks, ctlplane, 0]} - CTLPLANEHOST: {get_attr: [NetHostMap, value, ctlplane, short]} + - str_replace: + template: CTLPLANEIP CTLPLANEHOST.DOMAIN CTLPLANEHOST + params: + DOMAIN: {get_param: CloudDomain} + CTLPLANEIP: {get_attr: [{{server_resource_name}}, networks, ctlplane, 0]} + CTLPLANEHOST: {get_attr: [NetHostMap, value, ctlplane, short]} nova_server_resource: description: Heat resource handle for {{role.name}} server value: