restore missing fluentd client functionality

in the great rebase following the JINJA ALL THE THINGS changes we lost
critical functionality in the fluentd client service.  This review
restores the missing features.

Change-Id: I7c23f16f81e75f3da6a24587b2eb8385b3e920a4
Closes-bug: 1630692

overcloud.j2.yaml

@@ -292,8 +292,6 @@ resources:
                     services: {get_attr: [{{role.name}}ServiceChain, role_data, service_names]}
           ServiceNames: {get_attr: [{{role.name}}ServiceChain, role_data, service_names]}
           MonitoringSubscriptions: {get_attr: [{{role.name}}ServiceChain, role_data, monitoring_subscriptions]}
-          LoggingSources: {get_attr: [{{role.name}}ServiceChain, role_data, logging_sources]}
-          LoggingGroups: {get_attr: [{{role.name}}ServiceChain, role_data, logging_groups]}
 {% endfor %}
 
   allNodesConfig:
@@ -315,6 +313,24 @@ resources:
           - ','
 {% for role in roles %}
           - {get_attr: [{{role.name}}ServiceChain, role_data, service_names]}
+{% endfor %}
+      logging_groups:
+        yaql:
+          expression: >
+            $.data.groups.flatten()
+          data:
+            groups:
+{% for role in roles %}
+              - {get_attr: [{{role.name}}ServiceChain, role_data, logging_groups]}
+{% endfor %}
+      logging_sources:
+        yaql:
+          expression: >
+            $.data.sources.flatten()
+          data:
+            sources:
+{% for role in roles %}
+              - {get_attr: [{{role.name}}ServiceChain, role_data, logging_sources]}
 {% endfor %}
       controller_ips: {get_attr: [Controller, ip_address]}
       controller_names: {get_attr: [Controller, hostname]}

puppet/all-nodes-config.yaml

@@ -20,6 +20,10 @@ parameters:
     type: string
   controller_ips:
     type: comma_delimited_list
+  logging_groups:
+    type: json
+  logging_sources:
+    type: json
   service_ips:
     type: json
   service_node_names:
@@ -79,6 +83,8 @@ resources:
             all_nodes:
               mapped_data:
                 map_merge:
+                  - tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: logging_sources}
+                  - tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: logging_groups}
                   - enabled_services: {get_param: enabled_services}
                   # This writes out a mapping of service_name_enabled: 'true'
                   # For any services not enabled, hiera foo_enabled will

puppet/blockstorage-role.yaml

@@ -96,12 +96,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 resources:
   BlockStorage:
@@ -277,8 +271,6 @@ resources:
             volume:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Resource for site-specific injection of root certificate
   NodeTLSCAData:

puppet/cephstorage-role.yaml

@@ -102,12 +102,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 resources:
   CephStorage:
@@ -281,8 +275,6 @@ resources:
             ceph:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Resource for site-specific injection of root certificate
   NodeTLSCAData:

puppet/compute-role.yaml

@@ -114,12 +114,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 resources:
 
@@ -295,8 +289,6 @@ resources:
             compute:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   NovaComputeDeployment:
     type: OS::TripleO::SoftwareDeployment

puppet/controller-role.yaml

@@ -128,12 +128,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 parameter_groups:
 - label: deprecated
@@ -355,8 +349,6 @@ resources:
                 # Misc
                 tripleo::haproxy::service_certificate: {get_attr: [NodeTLSData, deployed_ssl_certificate_path]}
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Hook for site-specific additional pre-deployment config, e.g extra hieradata
   ControllerExtraConfigPre:

puppet/objectstorage-role.yaml

@@ -96,12 +96,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 resources:
 
@@ -266,8 +260,6 @@ resources:
             object:
               mapped_data: # data supplied directly to this deployment configuration, etc
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
 
   SwiftStorageHieraDeploy:

puppet/services/logging/fluentd-config.yaml

@@ -70,7 +70,7 @@ parameters:
       - tag_pattern: '**'
         type: record_transformer
         record:
-          nodename: '${hostname}'
+          host: '${hostname}'
 
       - tag_pattern: 'openstack.**'
         type: record_transformer

puppet/services/services.yaml

@@ -78,7 +78,7 @@ outputs:
             sources:
               - {get_attr: [LoggingConfiguration, LoggingDefaultSources]}
               - yaql:
-                  expression: list($.data.where($ != null).select($.get('logging_sources')).where($ != null))
+                  expression: list($.data.where($ != null).select($.get('logging_source')).where($ != null))
                   data: {get_attr: [ServiceChain, role_data]}
               - {get_attr: [LoggingConfiguration, LoggingExtraSources]}
             default_format: {get_attr: [LoggingConfiguration, LoggingDefaultFormat]}