Convert overcloud.yaml to support jinja2 templating

The first step of generating the Service chain resources via j2,
we'll then incrementally convert other resources to be created
in a similar way.

Partially-Implements: blueprint custom-roles
Depends-On: I81239991f36ed5f6453184bf9cffe930832cb68b
Change-Id: Iafa9b2afddf18a5a9833ec472a552fb256338b38

overcloud.j2.yaml

@@ -120,171 +120,33 @@ parameters:
     description: What interface to add to the HypervisorNeutronPhysicalBridge.
     type: string
 
-  ControllerServices:
+  # Jinja loop for Role in role_data.yaml
-    default:
+{% for role in roles %}
-      - OS::TripleO::Services::CACerts
+  # Resources generated for {{role.name}} Role
-      - OS::TripleO::Services::CephMon
+  {{role.name}}Services:
-      - OS::TripleO::Services::CephExternal
-      - OS::TripleO::Services::CinderApi
-      - OS::TripleO::Services::CinderBackup
-      - OS::TripleO::Services::CinderScheduler
-      - OS::TripleO::Services::CinderVolume
-      - OS::TripleO::Services::Core
-      - OS::TripleO::Services::Kernel
-      - OS::TripleO::Services::Keystone
-      - OS::TripleO::Services::GlanceApi
-      - OS::TripleO::Services::GlanceRegistry
-      - OS::TripleO::Services::HeatApi
-      - OS::TripleO::Services::HeatApiCfn
-      - OS::TripleO::Services::HeatApiCloudwatch
-      - OS::TripleO::Services::HeatEngine
-      - OS::TripleO::Services::MySQL
-      - OS::TripleO::Services::NeutronDhcpAgent
-      - OS::TripleO::Services::NeutronL3Agent
-      - OS::TripleO::Services::NeutronMetadataAgent
-      - OS::TripleO::Services::NeutronApi
-      - OS::TripleO::Services::NeutronCorePlugin
-      - OS::TripleO::Services::NeutronOvsAgent
-      - OS::TripleO::Services::RabbitMQ
-      - OS::TripleO::Services::HAproxy
-      - OS::TripleO::Services::Keepalived
-      - OS::TripleO::Services::Memcached
-      - OS::TripleO::Services::Pacemaker
-      - OS::TripleO::Services::Redis
-      - OS::TripleO::Services::NovaConductor
-      - OS::TripleO::Services::MongoDb
-      - OS::TripleO::Services::NovaApi
-      - OS::TripleO::Services::NovaScheduler
-      - OS::TripleO::Services::NovaConsoleauth
-      - OS::TripleO::Services::NovaVncproxy
-      - OS::TripleO::Services::Ntp
-      - OS::TripleO::Services::SwiftProxy
-      - OS::TripleO::Services::SwiftStorage
-      - OS::TripleO::Services::SwiftRingBuilder
-      - OS::TripleO::Services::Snmp
-      - OS::TripleO::Services::Timezone
-      - OS::TripleO::Services::CeilometerApi
-      - OS::TripleO::Services::CeilometerCollector
-      - OS::TripleO::Services::CeilometerExpirer
-      - OS::TripleO::Services::CeilometerAgentCentral
-      - OS::TripleO::Services::CeilometerAgentNotification
-      - OS::TripleO::Services::Horizon
-      - OS::TripleO::Services::GnocchiApi
-      - OS::TripleO::Services::GnocchiMetricd
-      - OS::TripleO::Services::GnocchiStatsd
-      - OS::Tripleo::Services::ManilaApi
-      - OS::Tripleo::Services::ManilaScheduler
-      - OS::Tripleo::Services::ManilaShare
-      - OS::TripleO::Services::AodhApi
-      - OS::TripleO::Services::AodhEvaluator
-      - OS::TripleO::Services::AodhNotifier
-      - OS::TripleO::Services::AodhListener
-      - OS::TripleO::Services::SaharaApi
-      - OS::TripleO::Services::SaharaEngine
-      - OS::TripleO::Services::IronicApi
-      - OS::TripleO::Services::IronicConductor
-      - OS::TripleO::Services::NovaIronic
-      - OS::TripleO::Services::TripleoPackages
-      - OS::TripleO::Services::TripleoFirewall
-      - OS::TripleO::Services::OpenDaylight
-      - OS::TripleO::Services::SensuClient
-      - OS::TripleO::Services::VipHosts
     description: A list of service resources (configured in the Heat
                  resource_registry) which represent nested stacks
-                 for each service that should get installed on the Controllers.
+                 for each service that should get installed on the {{role.name}} role.
-    type: comma_delimited_list
-
-  ComputeServices:
-    default:
-      - OS::TripleO::Services::CACerts
-      - OS::TripleO::Services::CephClient
-      - OS::TripleO::Services::CephExternal
-      - OS::TripleO::Services::Timezone
-      - OS::TripleO::Services::Ntp
-      - OS::TripleO::Services::Snmp
-      - OS::TripleO::Services::NovaCompute
-      - OS::TripleO::Services::NovaLibvirt
-      - OS::TripleO::Services::Kernel
-      - OS::TripleO::Services::ComputeNeutronCorePlugin
-      - OS::TripleO::Services::ComputeNeutronOvsAgent
-      - OS::TripleO::Services::ComputeCeilometerAgent
-      - OS::TripleO::Services::ComputeNeutronL3Agent
-      - OS::TripleO::Services::ComputeNeutronMetadataAgent
-      - OS::TripleO::Services::TripleoPackages
-      - OS::TripleO::Services::TripleoFirewall
-      - OS::TripleO::Services::NeutronSriovAgent
-      - OS::TripleO::Services::OpenDaylightOvs
-      - OS::TripleO::Services::SensuClient
-      - OS::TripleO::Services::VipHosts
-    description: A list of service resources (configured in the Heat
-                 resource_registry) which represent nested stacks
-                 for each service that should get installed on the Compute Nodes.
     type: comma_delimited_list
+  {% if role.ServicesDefault %}
+    default: {{role.ServicesDefault}}
+  {% endif %}
+{% endfor %}
 
 # Block storage specific parameters
   BlockStorageCount:
     type: number
     default: 0
-  BlockStorageServices:
-    default:
-      - OS::TripleO::Services::CACerts
-      - OS::TripleO::Services::CinderVolume
-      - OS::TripleO::Services::Kernel
-      - OS::TripleO::Services::Ntp
-      - OS::TripleO::Services::Timezone
-      - OS::TripleO::Services::Snmp
-      - OS::TripleO::Services::TripleoPackages
-      - OS::TripleO::Services::TripleoFirewall
-      - OS::TripleO::Services::SensuClient
-      - OS::TripleO::Services::VipHosts
-    description: A list of service resources (configured in the Heat
-                 resource_registry) which represent nested stacks
-                 for each service that should get installed on the BlockStorage nodes.
-    type: comma_delimited_list
 
 # Object storage specific parameters
   ObjectStorageCount:
     type: number
     default: 0
-  ObjectStorageServices:
-    default:
-      - OS::TripleO::Services::CACerts
-      - OS::TripleO::Services::Kernel
-      - OS::TripleO::Services::Ntp
-      - OS::TripleO::Services::SwiftStorage
-      - OS::TripleO::Services::SwiftRingBuilder
-      - OS::TripleO::Services::Snmp
-      - OS::TripleO::Services::Timezone
-      - OS::TripleO::Services::TripleoPackages
-      - OS::TripleO::Services::TripleoFirewall
-      - OS::TripleO::Services::SensuClient
-      - OS::TripleO::Services::VipHosts
-    description: A list of service resources (configured in the Heat
-                 resource_registry) which represent nested stacks
-                 for each service that should get installed on the ObjectStorage nodes.
-                 Note this role currently only supports steps 2, 3 and 4 configuration.
-    type: comma_delimited_list
-
 
 # Ceph storage specific parameters
   CephStorageCount:
     type: number
     default: 0
-  CephStorageServices:
-    default:
-      - OS::TripleO::Services::CACerts
-      - OS::TripleO::Services::CephOSD
-      - OS::TripleO::Services::Kernel
-      - OS::TripleO::Services::Ntp
-      - OS::TripleO::Services::Timezone
-      - OS::TripleO::Services::TripleoPackages
-      - OS::TripleO::Services::TripleoFirewall
-      - OS::TripleO::Services::SensuClient
-      - OS::TripleO::Services::VipHosts
-    description: A list of service resources (configured in the Heat
-                 resource_registry) which represent nested stacks
-                 for each service that should get installed on the CephStorage nodes.
-    type: comma_delimited_list
 
   # Hostname format for each role
   # Note %index% is translated into the index of the node, e.g 0/1/2 etc
@@ -390,13 +252,18 @@ resources:
       NetIpMap: {get_attr: [VipMap, net_ip_map]}
       ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
 
-  ControllerServiceChain:
+  # Jinja loop for Role in role_data.yaml
+{% for role in roles %}
+  # Resources generated for {{role.name}} Role
+  {{role.name}}ServiceChain:
     type: OS::TripleO::Services
     properties:
-      Services: {get_param: ControllerServices}
+      Services:
+        get_param: {{role.name}}Services
       ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
       EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
       DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
+{% endfor %}
 
   Controller:
     type: OS::Heat::ResourceGroup
@@ -428,14 +295,6 @@ resources:
           ServiceNames: {get_attr: [ControllerServiceChain, role_data, service_names]}
           MonitoringSubscriptions: {get_attr: [ControllerServiceChain, role_data, monitoring_subscriptions]}
 
-  ComputeServiceChain:
-    type: OS::TripleO::Services
-    properties:
-      Services: {get_param: ComputeServices}
-      ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
-      EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
-      DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
-
   Compute:
     type: OS::Heat::ResourceGroup
     depends_on: Networks
@@ -467,14 +326,6 @@ resources:
           ServiceNames: {get_attr: [ComputeServiceChain, role_data, service_names]}
           MonitoringSubscriptions: {get_attr: [ComputeServiceChain, role_data, monitoring_subscriptions]}
 
-  BlockStorageServiceChain:
-    type: OS::TripleO::Services
-    properties:
-      Services: {get_param: BlockStorageServices}
-      ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
-      EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
-      DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
-
   BlockStorage:
     type: OS::Heat::ResourceGroup
     depends_on: Networks
@@ -505,14 +356,6 @@ resources:
           ServiceNames: {get_attr: [BlockStorageServiceChain, role_data, service_names]}
           MonitoringSubscriptions: {get_attr: [BlockStorageServiceChain, role_data, monitoring_subscriptions]}
 
-  ObjectStorageServiceChain:
-    type: OS::TripleO::Services
-    properties:
-      Services: {get_param: ObjectStorageServices}
-      ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
-      EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
-      DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
-
   ObjectStorage:
     type: OS::Heat::ResourceGroup
     depends_on: Networks
@@ -543,14 +386,6 @@ resources:
           ServiceNames: {get_attr: [ObjectStorageServiceChain, role_data, service_names]}
           MonitoringSubscriptions: {get_attr: [ObjectStorageServiceChain, role_data, monitoring_subscriptions]}
 
-  CephStorageServiceChain:
-    type: OS::TripleO::Services
-    properties:
-      Services: {get_param: CephStorageServices}
-      ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
-      EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
-      DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
-
   CephStorage:
     type: OS::Heat::ResourceGroup
     depends_on: Networks

roles_data.yaml

@@ -0,0 +1,131 @@
+- name: Controller
+  ServicesDefault:
+    - OS::TripleO::Services::CACerts
+    - OS::TripleO::Services::CephMon
+    - OS::TripleO::Services::CephExternal
+    - OS::TripleO::Services::CinderApi
+    - OS::TripleO::Services::CinderBackup
+    - OS::TripleO::Services::CinderScheduler
+    - OS::TripleO::Services::CinderVolume
+    - OS::TripleO::Services::Core
+    - OS::TripleO::Services::Kernel
+    - OS::TripleO::Services::Keystone
+    - OS::TripleO::Services::GlanceApi
+    - OS::TripleO::Services::GlanceRegistry
+    - OS::TripleO::Services::HeatApi
+    - OS::TripleO::Services::HeatApiCfn
+    - OS::TripleO::Services::HeatApiCloudwatch
+    - OS::TripleO::Services::HeatEngine
+    - OS::TripleO::Services::MySQL
+    - OS::TripleO::Services::NeutronDhcpAgent
+    - OS::TripleO::Services::NeutronL3Agent
+    - OS::TripleO::Services::NeutronMetadataAgent
+    - OS::TripleO::Services::NeutronApi
+    - OS::TripleO::Services::NeutronCorePlugin
+    - OS::TripleO::Services::NeutronOvsAgent
+    - OS::TripleO::Services::RabbitMQ
+    - OS::TripleO::Services::HAproxy
+    - OS::TripleO::Services::Keepalived
+    - OS::TripleO::Services::Memcached
+    - OS::TripleO::Services::Pacemaker
+    - OS::TripleO::Services::Redis
+    - OS::TripleO::Services::NovaConductor
+    - OS::TripleO::Services::MongoDb
+    - OS::TripleO::Services::NovaApi
+    - OS::TripleO::Services::NovaScheduler
+    - OS::TripleO::Services::NovaConsoleauth
+    - OS::TripleO::Services::NovaVncproxy
+    - OS::TripleO::Services::Ntp
+    - OS::TripleO::Services::SwiftProxy
+    - OS::TripleO::Services::SwiftStorage
+    - OS::TripleO::Services::SwiftRingBuilder
+    - OS::TripleO::Services::Snmp
+    - OS::TripleO::Services::Timezone
+    - OS::TripleO::Services::CeilometerApi
+    - OS::TripleO::Services::CeilometerCollector
+    - OS::TripleO::Services::CeilometerExpirer
+    - OS::TripleO::Services::CeilometerAgentCentral
+    - OS::TripleO::Services::CeilometerAgentNotification
+    - OS::TripleO::Services::Horizon
+    - OS::TripleO::Services::GnocchiApi
+    - OS::TripleO::Services::GnocchiMetricd
+    - OS::TripleO::Services::GnocchiStatsd
+    - OS::Tripleo::Services::ManilaApi
+    - OS::Tripleo::Services::ManilaScheduler
+    - OS::Tripleo::Services::ManilaShare
+    - OS::TripleO::Services::AodhApi
+    - OS::TripleO::Services::AodhEvaluator
+    - OS::TripleO::Services::AodhNotifier
+    - OS::TripleO::Services::AodhListener
+    - OS::TripleO::Services::SaharaApi
+    - OS::TripleO::Services::SaharaEngine
+    - OS::TripleO::Services::IronicApi
+    - OS::TripleO::Services::IronicConductor
+    - OS::TripleO::Services::NovaIronic
+    - OS::TripleO::Services::TripleoPackages
+    - OS::TripleO::Services::TripleoFirewall
+    - OS::TripleO::Services::OpenDaylight
+    - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::VipHosts
+
+- name: Compute
+  ServicesDefault:
+    - OS::TripleO::Services::CACerts
+    - OS::TripleO::Services::CephClient
+    - OS::TripleO::Services::CephExternal
+    - OS::TripleO::Services::Timezone
+    - OS::TripleO::Services::Ntp
+    - OS::TripleO::Services::Snmp
+    - OS::TripleO::Services::NovaCompute
+    - OS::TripleO::Services::NovaLibvirt
+    - OS::TripleO::Services::Kernel
+    - OS::TripleO::Services::ComputeNeutronCorePlugin
+    - OS::TripleO::Services::ComputeNeutronOvsAgent
+    - OS::TripleO::Services::ComputeCeilometerAgent
+    - OS::TripleO::Services::ComputeNeutronL3Agent
+    - OS::TripleO::Services::ComputeNeutronMetadataAgent
+    - OS::TripleO::Services::TripleoPackages
+    - OS::TripleO::Services::TripleoFirewall
+    - OS::TripleO::Services::NeutronSriovAgent
+    - OS::TripleO::Services::OpenDaylightOvs
+    - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::VipHosts
+
+- name: BlockStorage
+  ServicesDefault:
+    - OS::TripleO::Services::CACerts
+    - OS::TripleO::Services::CinderVolume
+    - OS::TripleO::Services::Kernel
+    - OS::TripleO::Services::Ntp
+    - OS::TripleO::Services::Timezone
+    - OS::TripleO::Services::Snmp
+    - OS::TripleO::Services::TripleoPackages
+    - OS::TripleO::Services::TripleoFirewall
+    - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::VipHosts
+
+- name: ObjectStorage
+  ServicesDefault:
+    - OS::TripleO::Services::CACerts
+    - OS::TripleO::Services::Kernel
+    - OS::TripleO::Services::Ntp
+    - OS::TripleO::Services::SwiftStorage
+    - OS::TripleO::Services::SwiftRingBuilder
+    - OS::TripleO::Services::Snmp
+    - OS::TripleO::Services::Timezone
+    - OS::TripleO::Services::TripleoPackages
+    - OS::TripleO::Services::TripleoFirewall
+    - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::VipHosts
+
+- name: CephStorage
+  ServicesDefault:
+    - OS::TripleO::Services::CACerts
+    - OS::TripleO::Services::CephOSD
+    - OS::TripleO::Services::Kernel
+    - OS::TripleO::Services::Ntp
+    - OS::TripleO::Services::Timezone
+    - OS::TripleO::Services::TripleoPackages
+    - OS::TripleO::Services::TripleoFirewall
+    - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::VipHosts

tools/yaml-validate.py

@@ -64,15 +64,16 @@ def validate(filename):
         print(traceback.format_exc())
         return 1
     # yaml is OK, now walk the parameters and output a warning for unused ones
-    for p in tpl.get('parameters', {}):
+    if 'heat_template_version' in tpl:
-        if p in required_params:
+        for p in tpl.get('parameters', {}):
-            continue
+            if p in required_params:
-        str_p = '\'%s\'' % p
+                continue
-        in_resources = str_p in str(tpl.get('resources', {}))
+            str_p = '\'%s\'' % p
-        in_outputs = str_p in str(tpl.get('outputs', {}))
+            in_resources = str_p in str(tpl.get('resources', {}))
-        if not in_resources and not in_outputs:
+            in_outputs = str_p in str(tpl.get('outputs', {}))
-            print('Warning: parameter %s in template %s appears to be unused'
+            if not in_resources and not in_outputs:
-                  % (p, filename))
+                print('Warning: parameter %s in template %s '
+                      'appears to be unused' % (p, filename))
 
     return retval
 
@@ -87,7 +88,7 @@ for base_path in path_args:
     if os.path.isdir(base_path):
         for subdir, dirs, files in os.walk(base_path):
             for f in files:
-                if f.endswith('.yaml'):
+                if f.endswith('.yaml') and not f.endswith('.j2.yaml'):
                     file_path = os.path.join(subdir, f)
                     failed = validate(file_path)
                     if failed: