Make krb-service-principal metadata per-Role

Not all roles are connected to all networks, there is no
need to create metadata for networks not associated with
the role.

In edge/spine-and-leaf deployments the total number of
composable networks used can be high. Passing all the
networks we quickly go beyond the nova metadata fields
size limit (each field cannot exceed 256 bytes).

Also update tools/check-up-to-date.sh script to use the
simple yaml-diff.py instead of diff. The env generator
code will sort data, while jinja rendered environments
are not sorted, thus need to diff the data in yaml not
the text.

Conflicts:
    environments/ssl/enable-internal-tls.j2.yaml
    extraconfig/nova_metadata/krb-service-principals/role.role.j2.yaml

Closes-Bug: #1821377
Change-Id: I5ae3bc845b0a6ad6986d44b14ff4b0737a9b033b
(cherry picked from commit d5ecc1f651)

common/services/role.role.j2.yaml

@@ -58,7 +58,7 @@ resources:
     type: OS::TripleO::LoggingConfiguration
 
   ServiceServerMetadataHook:
-    type: OS::TripleO::ServiceServerMetadataHook
+    type: OS::TripleO::{{role.name}}ServiceServerMetadataHook
     properties:
       RoleData: {get_attr: [ServiceChain, role_data]}
 
@@ -70,7 +70,7 @@ resources:
         list_join:
           - "\n"
           - - str_replace:
-                template: {get_file: ../puppet/manifests/overcloud_common.pp}
+                template: {get_file: ../../puppet/manifests/overcloud_common.pp}
                 params:
                   __ROLE__: {get_param: RoleName}
             - yaql:

environments/ssl/enable-internal-tls.j2.yaml

@@ -30,8 +30,10 @@ parameter_defaults:
   # End static parameters
   # *********************
 resource_registry:
-  OS::TripleO::ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals.yaml
   OS::TripleO::Services::CertmongerUser: ../../puppet/services/certmonger-user.yaml
   OS::TripleO::Services::HAProxyInternalTLS: ../../puppet/services/haproxy-internal-tls-certmonger.yaml
   OS::TripleO::Services::IpaClient: ../../extraconfig/services/ipaclient.yaml
   OS::TripleO::Services::TLSProxyBase: ../../puppet/services/apache.yaml
+{%- for role in roles %}
+  OS::TripleO::{{role.name}}ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/{{role.name.lower()}}-role.yaml
+{%- endfor %}

extraconfig/nova_metadata/krb-service-principals/role.role.j2.yaml

@@ -1,11 +1,11 @@
 heat_template_version: queens
-description: 'Generates the relevant service principals for a server'
+description: 'Generates the relevant service principals for a {{role.name}} server'
 
 parameters:
   RoleData:
      type: json
      description: the list containing the 'role_data' output for the ServiceChain
-{%- for network in networks if network.vip|default(false) %}
+{%- for network in networks if network.vip|default(false) and network.name in role.networks %}
 {%- if network.name == 'External' %}
   # Special case the External hostname param, which is CloudName
   CloudName:
@@ -69,7 +69,7 @@ resources:
           data:
             metadata: {get_attr: [IncomingMetadataSettings, value]}
             fqdns:
-{%- for network in networks if network.vip|default(false) %}
+{%- for network in networks if network.vip|default(false) and network.name in role.networks %}
 {%- if network.name == 'External' %}
               external: {get_param: CloudName}
 {%- elif network.name == 'InternalApi' %}
@@ -97,4 +97,3 @@ outputs:
       map_merge:
         - {get_attr: [IndividualServices, value]}
         - {get_attr: [CompactServices, value]}
-

overcloud-resource-registry-puppet.j2.yaml

@@ -42,7 +42,9 @@ resource_registry:
   # in the jinja loop
   OS::TripleO::Controller::Net::SoftwareConfig: net-config-bridge.yaml
 
-  OS::TripleO::ServiceServerMetadataHook: OS::Heat::None
+{% for role in roles %}
+  OS::TripleO::{{role.name}}ServiceServerMetadataHook: OS::Heat::None
+{%- endfor %}
 
   OS::TripleO::Server: OS::Nova::Server
 {% for role in roles %}
@@ -111,7 +113,9 @@ resource_registry:
   OS::TripleO::WorkflowSteps: OS::Mistral::ExternalResource
 
   # services
-  OS::TripleO::Services: common/services.yaml
+{%- for role in roles %}
+  OS::TripleO::{{role.name}}Services: common/services/{{role.name.lower()}}-role.yaml
+{%- endfor %}
   OS::TripleO::Services::Aide: OS::Heat::None
   OS::TripleO::Services::Apache: puppet/services/apache.yaml
   OS::TripleO::Services::CACerts: puppet/services/ca-certs.yaml

overcloud.j2.yaml

@@ -391,7 +391,7 @@ resources:
 {% for role in roles %}
   # Resources generated for {{role.name}} Role
   {{role.name}}ServiceChain:
-    type: OS::TripleO::Services
+    type: OS::TripleO::{{role.name}}Services
     properties:
       Services:
         get_param: {{role.name}}Services

sample-env-generator/ssl.yaml

@@ -62,7 +62,11 @@ environments:
       OS::TripleO::Services::TLSProxyBase: ../../puppet/services/apache.yaml
       # Creates nova metadata that will create the extra service principals per
       # node.
-      OS::TripleO::ServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals.yaml
+      OS::TripleO::ControllerServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/controller-role.yaml
+      OS::TripleO::ComputeServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/compute-role.yaml
+      OS::TripleO::BlockStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/blockstorage-role.yaml
+      OS::TripleO::ObjectStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/objectstorage-role.yaml
+      OS::TripleO::CephStorageServiceServerMetadataHook: ../../extraconfig/nova_metadata/krb-service-principals/cephstorage-role.yaml
   - name: ssl/inject-trust-anchor
     title: Inject SSL Trust Anchor on Overcloud Nodes
     description: |

tools/check-up-to-date.sh

@@ -17,7 +17,7 @@ cd $tmpdir
 
 file_list=$(find environments -type f)
 for f in $file_list; do
-    if ! diff -q $f $base/$f; then
+    if ! $base/tools/yaml-diff.py $f $base/$f; then
         echo "ERROR: $base/$f is not up to date"
         diff $f $base/$f
         retval=1

tools/process-templates.py

@@ -311,9 +311,16 @@ def clean_templates(base_path, role_data_path, network_data_path):
         host_config_and_reboot_path = os.path.join(
             'extraconfig', 'pre_network',
             '%s-host_config_and_reboot.yaml' % role['name'].lower())
+        krb_service_principals_path = os.path.join(
+            'extraconfig', 'nova_metadata', 'krb-service-principals',
+            '%s-role.yaml' % role['name'].lower())
+        common_services_path = os.path.join(
+            'common', 'services', '%s-role.yaml' % role['name'].lower())
 
         delete(role_path)
         delete(host_config_and_reboot_path)
+        delete(krb_service_principals_path)
+        delete(common_services_path)
 
 opts = parse_opts(sys.argv)
 

tools/yaml-diff.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sys
+import yaml
+
+if len(sys.argv) != 3:
+    raise RuntimeError('Not enough arguemnts')
+
+FILE_A = sys.argv[1]
+FILE_B = sys.argv[2]
+
+with open(FILE_A, 'r') as file_a:
+    a = yaml.safe_load(file_a)
+
+with open(FILE_B, 'r') as file_b:
+    b = yaml.safe_load(file_b)
+
+if a != b:
+    sys.exit("Files are different")
+
+sys.exit(0)