From d5ecc1f6518d14f1ae6fefebd73533f11762bedf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harald=20Jens=C3=A5s?= Date: Sat, 23 Mar 2019 14:50:27 +0100 Subject: [PATCH] Make krb-service-principal metadata per-Role Not all roles are connected to all networks, there is no need to create metadata for networks not associated with the role. In edge/spine-and-leaf deployments the total number of composable networks used can be high. Passing all the networks we quickly go beyond the nova metadata fields size limit (each field cannot exceed 256 bytes). Also update tools/check-up-to-date.sh script to use the simple yaml-diff.py instead of diff. The env generator code will sort data, while jinja rendered environments are not sorted, thus need to diff the data in yaml not the text. Closes-Bug: #1821377 Change-Id: I5ae3bc845b0a6ad6986d44b14ff4b0737a9b033b --- .../role.role.j2.yaml} | 4 +-- ...l-tls.yaml => enable-internal-tls.j2.yaml} | 4 ++- .../role.role.j2.yaml} | 7 ++-- overcloud-resource-registry-puppet.j2.yaml | 8 +++-- overcloud.j2.yaml | 2 +- sample-env-generator/ssl.yaml | 6 +++- tools/check-up-to-date.sh | 2 +- tools/process-templates.py | 7 ++++ tools/yaml-diff.py | 32 +++++++++++++++++++ 9 files changed, 60 insertions(+), 12 deletions(-) rename common/{services.yaml => services/role.role.j2.yaml} (99%) rename environments/ssl/{enable-internal-tls.yaml => enable-internal-tls.j2.yaml} (90%) rename extraconfig/nova_metadata/{krb-service-principals.j2.yaml => krb-service-principals/role.role.j2.yaml} (93%) create mode 100755 tools/yaml-diff.py diff --git a/common/services.yaml b/common/services/role.role.j2.yaml similarity index 99% rename from common/services.yaml rename to common/services/role.role.j2.yaml index e092ecdf13..c033cf88ca 100644 --- a/common/services.yaml +++ b/common/services/role.role.j2.yaml @@ -58,7 +58,7 @@ resources: type: OS::TripleO::LoggingConfiguration ServiceServerMetadataHook: - type: OS::TripleO::ServiceServerMetadataHook + type: OS::TripleO::{{role.name}}ServiceServerMetadataHook properties: RoleData: {get_attr: [ServiceChain, role_data]} @@ -70,7 +70,7 @@ resources: list_join: - "\n" - - str_replace: - template: {get_file: ../puppet/manifests/overcloud_common.pp} + template: {get_file: ../../puppet/manifests/overcloud_common.pp} params: __ROLE__: {get_param: RoleName} - yaql: diff --git a/environments/ssl/enable-internal-tls.yaml b/environments/ssl/enable-internal-tls.j2.yaml similarity index 90% rename from environments/ssl/enable-internal-tls.yaml rename to environments/ssl/enable-internal-tls.j2.yaml index 7962b3179b..f6208f46f9 100644 --- a/environments/ssl/enable-internal-tls.yaml +++ b/environments/ssl/enable-internal-tls.j2.yaml @@ -34,8 +34,10 @@ parameter_defaults: # End static parameters # ********************* resource_registry: - OS::TripleO::ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals.yaml OS::TripleO::Services::CertmongerUser: ../../puppet/services/certmonger-user.yaml OS::TripleO::Services::HAProxyInternalTLS: ../../deployment/haproxy/haproxy-internal-tls-certmonger.yaml OS::TripleO::Services::IpaClient: ../../extraconfig/services/ipaclient.yaml OS::TripleO::Services::TLSProxyBase: ../../deployment/apache/apache-baremetal-puppet.yaml +{%- for role in roles %} + OS::TripleO::{{role.name}}ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/{{role.name.lower()}}-role.yaml +{%- endfor %} diff --git a/extraconfig/nova_metadata/krb-service-principals.j2.yaml b/extraconfig/nova_metadata/krb-service-principals/role.role.j2.yaml similarity index 93% rename from extraconfig/nova_metadata/krb-service-principals.j2.yaml rename to extraconfig/nova_metadata/krb-service-principals/role.role.j2.yaml index 633a5c0838..8f0edac25f 100644 --- a/extraconfig/nova_metadata/krb-service-principals.j2.yaml +++ b/extraconfig/nova_metadata/krb-service-principals/role.role.j2.yaml @@ -1,11 +1,11 @@ heat_template_version: rocky -description: 'Generates the relevant service principals for a server' +description: 'Generates the relevant service principals for a {{role.name}} server' parameters: RoleData: type: json description: the list containing the 'role_data' output for the ServiceChain -{%- for network in networks if network.vip|default(false) %} +{%- for network in networks if network.vip|default(false) and network.name in role.networks %} {%- if network.name == 'External' %} # Special case the External hostname param, which is CloudName CloudName: @@ -69,7 +69,7 @@ resources: data: metadata: {get_attr: [IncomingMetadataSettings, value]} fqdns: -{%- for network in networks if network.vip|default(false) %} +{%- for network in networks if network.vip|default(false) and network.name in role.networks %} {%- if network.name == 'External' %} external: {get_param: CloudName} {%- elif network.name == 'InternalApi' %} @@ -97,4 +97,3 @@ outputs: map_merge: - {get_attr: [IndividualServices, value]} - {get_attr: [CompactServices, value]} - diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml index 58549e1300..53165481bb 100644 --- a/overcloud-resource-registry-puppet.j2.yaml +++ b/overcloud-resource-registry-puppet.j2.yaml @@ -31,7 +31,9 @@ resource_registry: # in the jinja loop OS::TripleO::Controller::Net::SoftwareConfig: net-config-bridge.yaml - OS::TripleO::ServiceServerMetadataHook: OS::Heat::None +{% for role in roles %} + OS::TripleO::{{role.name}}ServiceServerMetadataHook: OS::Heat::None +{%- endfor %} OS::TripleO::Server: OS::Nova::Server {% for role in roles %} @@ -100,7 +102,9 @@ resource_registry: OS::TripleO::WorkflowSteps: OS::Mistral::ExternalResource # services - OS::TripleO::Services: common/services.yaml +{%- for role in roles %} + OS::TripleO::{{role.name}}Services: common/services/{{role.name.lower()}}-role.yaml +{%- endfor %} OS::TripleO::Services::Aide: OS::Heat::None OS::TripleO::Services::Apache: deployment/apache/apache-baremetal-puppet.yaml OS::TripleO::Services::CACerts: puppet/services/ca-certs.yaml diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml index 404252599c..08b2a5b532 100644 --- a/overcloud.j2.yaml +++ b/overcloud.j2.yaml @@ -427,7 +427,7 @@ resources: {% for role in roles %} # Resources generated for {{role.name}} Role {{role.name}}ServiceChain: - type: OS::TripleO::Services + type: OS::TripleO::{{role.name}}Services properties: Services: get_param: {{role.name}}Services diff --git a/sample-env-generator/ssl.yaml b/sample-env-generator/ssl.yaml index f1f60dc83f..f9a6348c0c 100644 --- a/sample-env-generator/ssl.yaml +++ b/sample-env-generator/ssl.yaml @@ -65,7 +65,11 @@ environments: OS::TripleO::Services::TLSProxyBase: ../../deployment/apache/apache-baremetal-puppet.yaml # Creates nova metadata that will create the extra service principals per # node. - OS::TripleO::ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals.yaml + OS::TripleO::ControllerServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/controller-role.yaml + OS::TripleO::ComputeServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/compute-role.yaml + OS::TripleO::BlockStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/blockstorage-role.yaml + OS::TripleO::ObjectStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/objectstorage-role.yaml + OS::TripleO::CephStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/cephstorage-role.yaml - name: ssl/inject-trust-anchor title: Inject SSL Trust Anchor on Overcloud Nodes description: | diff --git a/tools/check-up-to-date.sh b/tools/check-up-to-date.sh index eaa29d31c1..41a9c9f24f 100755 --- a/tools/check-up-to-date.sh +++ b/tools/check-up-to-date.sh @@ -18,7 +18,7 @@ cd $tmpdir file_list=$(find environments -type f) for f in $file_list; do - if ! diff -q $f $base/$f; then + if ! $base/tools/yaml-diff.py $f $base/$f; then echo "ERROR: $base/$f is not up to date" diff $f $base/$f retval=1 diff --git a/tools/process-templates.py b/tools/process-templates.py index 832148e64b..c820be4ee7 100755 --- a/tools/process-templates.py +++ b/tools/process-templates.py @@ -328,9 +328,16 @@ def clean_templates(base_path, role_data_path, network_data_path): host_config_and_reboot_path = os.path.join( 'extraconfig', 'pre_network', '%s-host_config_and_reboot.yaml' % role['name'].lower()) + krb_service_principals_path = os.path.join( + 'extraconfig', 'nova_metadata', 'krb-service-principals', + '%s-role.yaml' % role['name'].lower()) + common_services_path = os.path.join( + 'common', 'services', '%s-role.yaml' % role['name'].lower()) delete(role_path) delete(host_config_and_reboot_path) + delete(krb_service_principals_path) + delete(common_services_path) nic_config_dir = os.path.join(base_path, 'network', 'config') for sample_nic_config_dir in os.listdir(nic_config_dir): diff --git a/tools/yaml-diff.py b/tools/yaml-diff.py new file mode 100755 index 0000000000..6001daf0e1 --- /dev/null +++ b/tools/yaml-diff.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import yaml + +if len(sys.argv) != 3: + raise RuntimeError('Not enough arguemnts') + +FILE_A = sys.argv[1] +FILE_B = sys.argv[2] + +with open(FILE_A, 'r') as file_a: + a = yaml.safe_load(file_a) + +with open(FILE_B, 'r') as file_b: + b = yaml.safe_load(file_b) + +if a != b: + sys.exit("Files are different") + +sys.exit(0)