From b20bce1bf02889b05d07eb41ceef02affbc458a3 Mon Sep 17 00:00:00 2001
From: Lars Kellogg-Stedman <lars@redhat.com>
Date: Thu, 24 Aug 2017 21:29:44 -0700
Subject: [PATCH] logging: use service_config_settings for fluentd

The initial fluentd client implementation predates the introduction of
service_config_settings, and necessitated some invasive changes to
what is now common/serivces.yaml. This commit modifies existing
services to use the service_config_settings based configuration
mechanism supported by more recent versions of the fluentd support in
puppet-tripleo.

Partial-bug: #1715187
Depends-On: I3149902401d68d6fd236073a73a20f982d4b952a
Depends-On: I2b057190ec0e4e75ee4ee47ebe0164c2644e5ab7
Depends-On: Ie7df4b8b94cb0ae38096ab95800f211ef1cd8455
Change-Id: I28028ffa00df2da8e0478a551d3de89c3ee46e1f
---
 .../scenario001-multinode-containers.yaml       |  1 +
 puppet/services/ceilometer-agent-central.yaml   | 12 ++++++++----
 puppet/services/ceilometer-agent-ipmi.yaml      |  9 ++++++---
 .../services/ceilometer-agent-notification.yaml | 12 +++++++-----
 puppet/services/cinder-api.yaml                 |  8 +++++---
 puppet/services/cinder-scheduler.yaml           |  9 ++++++---
 puppet/services/cinder-volume.yaml              |  9 ++++++---
 puppet/services/database/mongodb.yaml           | 17 ++++++++++-------
 puppet/services/ec2-api.yaml                    |  8 +++++---
 puppet/services/glance-api.yaml                 |  8 +++++---
 puppet/services/gnocchi-api.yaml                |  8 +++++---
 puppet/services/heat-api-cfn.yaml               |  8 +++++---
 puppet/services/heat-api.yaml                   |  8 +++++---
 puppet/services/heat-engine.yaml                |  8 +++++---
 puppet/services/keystone.yaml                   |  8 +++++---
 puppet/services/neutron-api.yaml                |  8 +++++---
 puppet/services/neutron-dhcp.yaml               |  9 ++++++---
 puppet/services/neutron-l2gw-agent.yaml         |  9 ++++++---
 puppet/services/neutron-l3-compute-dvr.yaml     |  9 ++++++---
 puppet/services/neutron-l3.yaml                 |  9 ++++++---
 puppet/services/neutron-metadata.yaml           |  9 ++++++---
 puppet/services/neutron-ovs-agent.yaml          |  9 ++++++---
 puppet/services/nova-api.yaml                   |  8 +++++---
 puppet/services/nova-compute.yaml               |  8 +++++---
 puppet/services/nova-conductor.yaml             | 15 +++++++++------
 puppet/services/nova-consoleauth.yaml           |  9 ++++++---
 puppet/services/nova-placement.yaml             |  8 +++++---
 puppet/services/nova-scheduler.yaml             |  9 ++++++---
 puppet/services/nova-vnc-proxy.yaml             |  9 ++++++---
 puppet/services/octavia-api.yaml                |  8 +++++---
 puppet/services/octavia-health-manager.yaml     |  9 ++++++---
 puppet/services/octavia-housekeeping.yaml       |  9 ++++++---
 puppet/services/octavia-worker.yaml             |  9 ++++++---
 puppet/services/pacemaker.yaml                  | 10 ++++++----
 puppet/services/pacemaker/cinder-volume.yaml    |  2 --
 puppet/services/pacemaker_remote.yaml           |  9 ++++++---
 puppet/services/sahara-api.yaml                 |  9 ++++++---
 puppet/services/sahara-engine.yaml              |  9 ++++++---
 38 files changed, 214 insertions(+), 121 deletions(-)

diff --git a/ci/environments/scenario001-multinode-containers.yaml b/ci/environments/scenario001-multinode-containers.yaml
index 2d7a5445b5..11388248c3 100644
--- a/ci/environments/scenario001-multinode-containers.yaml
+++ b/ci/environments/scenario001-multinode-containers.yaml
@@ -20,6 +20,7 @@ resource_registry:
   OS::TripleO::Tasks::ControllerPreConfig: OS::Heat::None
   OS::TripleO::Tasks::ControllerPostConfig: OS::Heat::None
   OS::TripleO::Tasks::ControllerPostPuppetRestart: ../../extraconfig/tasks/post_puppet_pacemaker_restart.yaml
+  #Needs to run scenario001
   OS::TripleO::Services::Fluentd: ../../docker/services/fluentd.yaml
   OS::TripleO::Services::SensuClient: ../../docker/services/sensu-client.yaml
   # Some infra instances don't pass the ping test but are otherwise working.
diff --git a/puppet/services/ceilometer-agent-central.yaml b/puppet/services/ceilometer-agent-central.yaml
index c033a480cb..7612bee9a0 100644
--- a/puppet/services/ceilometer-agent-central.yaml
+++ b/puppet/services/ceilometer-agent-central.yaml
@@ -60,16 +60,20 @@ outputs:
     value:
       service_name: ceilometer_agent_central
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerCentral}
-      logging_source: {get_param: CeilometerAgentCentralLoggingSource}
-      logging_groups:
-        - ceilometer
       config_settings:
         map_merge:
           - get_attr: [CeilometerServiceBase, role_data, config_settings]
           - ceilometer_redis_password: {get_param: RedisPassword}
             central_namespace: true
       service_config_settings:
-        get_attr: [CeilometerServiceBase, role_data, service_config_settings]
+        map_merge:
+          - get_attr: [CeilometerServiceBase, role_data, service_config_settings]
+          - fluentd:
+              tripleo_fluentd_groups_ceilometer_agent_central:
+                - ceilometer
+              tripleo_fluentd_sources_ceilometer_agent_central:
+                - {get_param: CeilometerAgentCentralLoggingSource}
+
       step_config: |
         include ::tripleo::profile::base::ceilometer::agent::polling
       upgrade_tasks:
diff --git a/puppet/services/ceilometer-agent-ipmi.yaml b/puppet/services/ceilometer-agent-ipmi.yaml
index 236a3392b0..b4f0ad41fb 100644
--- a/puppet/services/ceilometer-agent-ipmi.yaml
+++ b/puppet/services/ceilometer-agent-ipmi.yaml
@@ -60,14 +60,17 @@ outputs:
     value:
       service_name: ceilometer_agent_ipmi
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerIpmi}
-      logging_source: {get_param: CeilometerAgentIpmiLoggingSource}
-      logging_groups:
-        - ceilometer
       config_settings:
         map_merge:
           - get_attr: [CeilometerServiceBase, role_data, config_settings]
           - ceilometer_redis_password: {get_param: RedisPassword}
             ipmi_namespace: true
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_ceilometer_agent_ipmi:
+            - ceilometer
+          tripleo_fluentd_sources_ceilometer_agent_ipmi:
+            - {get_param: CeilometerAgentIpmiLoggingSource}
       step_config: |
         include ::tripleo::profile::base::ceilometer::agent::polling
       upgrade_tasks:
diff --git a/puppet/services/ceilometer-agent-notification.yaml b/puppet/services/ceilometer-agent-notification.yaml
index 60e6fc4a82..d86a42b3d1 100644
--- a/puppet/services/ceilometer-agent-notification.yaml
+++ b/puppet/services/ceilometer-agent-notification.yaml
@@ -39,7 +39,6 @@ parameters:
       tag: openstack.ceilometer.agent.notification
       path: /var/log/ceilometer/agent-notification.log
 
-
 resources:
   CeilometerServiceBase:
     type: ./ceilometer-base.yaml
@@ -57,13 +56,16 @@ outputs:
     value:
       service_name: ceilometer_agent_notification
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerNotification}
-      logging_source: {get_param: CeilometerAgentNotificationLoggingSource}
-      logging_groups:
-        - ceilometer
       config_settings:
         get_attr: [CeilometerServiceBase, role_data, config_settings]
       service_config_settings:
-        get_attr: [CeilometerServiceBase, role_data, service_config_settings]
+        map_merge:
+          - get_attr: [CeilometerServiceBase, role_data, service_config_settings]
+          - fluentd:
+              tripleo_fluentd_groups_ceilometer_agent_notification:
+                - ceilometer
+              tripleo_fluentd_sources_ceilometer_agent_notification:
+                - {get_param: CeilometerAgentNotificationLoggingSource}
       step_config: |
         include ::tripleo::profile::base::ceilometer::agent::notification
       upgrade_tasks:
diff --git a/puppet/services/cinder-api.yaml b/puppet/services/cinder-api.yaml
index 1c8d4e6515..5ffcfa9f30 100644
--- a/puppet/services/cinder-api.yaml
+++ b/puppet/services/cinder-api.yaml
@@ -103,9 +103,6 @@ outputs:
     value:
       service_name: cinder_api
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderApi}
-      logging_source: {get_param: CinderApiLoggingSource}
-      logging_groups:
-        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]
@@ -160,6 +157,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::cinder::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_cinder_api:
+            - cinder
+          tripleo_fluentd_sources_cinder_api:
+            - {get_param: CinderApiLoggingSource}
         keystone:
           cinder::keystone::auth::tenant: 'service'
           cinder::keystone::auth::public_url: {get_param: [EndpointMap, CinderPublic, uri]}
diff --git a/puppet/services/cinder-scheduler.yaml b/puppet/services/cinder-scheduler.yaml
index 2ed04b5a10..1654059ae8 100644
--- a/puppet/services/cinder-scheduler.yaml
+++ b/puppet/services/cinder-scheduler.yaml
@@ -57,13 +57,16 @@ outputs:
     value:
       service_name: cinder_scheduler
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderScheduler}
-      logging_source: {get_param: CinderSchedulerLoggingSource}
-      logging_groups:
-        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]
           - cinder::scheduler::scheduler_driver: cinder.scheduler.filter_scheduler.FilterScheduler
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_cinder_scheduler:
+            - cinder
+          tripleo_fluentd_sources_cinder_scheduler:
+            - {get_param: CinderSchedulerLoggingSource}
       step_config: |
         include ::tripleo::profile::base::cinder::scheduler
       upgrade_tasks:
diff --git a/puppet/services/cinder-volume.yaml b/puppet/services/cinder-volume.yaml
index b4fa38f0be..c4e7d78e82 100644
--- a/puppet/services/cinder-volume.yaml
+++ b/puppet/services/cinder-volume.yaml
@@ -121,9 +121,6 @@ outputs:
     value:
       service_name: cinder_volume
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderVolume}
-      logging_source: {get_param: CinderVolumeLoggingSource}
-      logging_groups:
-        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]
@@ -149,6 +146,12 @@ outputs:
             # internal_api_uri -> [IP]
             # internal_api_subnet - > IP/CIDR
             tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_address: {get_param: [ServiceNetMap, CinderIscsiNetwork]}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_cinder_volume:
+            - cinder
+          tripleo_fluentd_sources_cinder_volume:
+            - {get_param: CinderVolumeLoggingSource}
       step_config: |
         include ::tripleo::profile::base::cinder::volume
       upgrade_tasks:
diff --git a/puppet/services/database/mongodb.yaml b/puppet/services/database/mongodb.yaml
index 4677a9f4ce..fef239b9cc 100644
--- a/puppet/services/database/mongodb.yaml
+++ b/puppet/services/database/mongodb.yaml
@@ -31,10 +31,6 @@ parameters:
     description: Mapping of service endpoint -> protocol. Typically set
                  via parameter_defaults in the resource registry.
     type: json
-  MongodbMemoryLimit:
-    default: '20G'
-    description: Limit the amount of memory mongodb uses with systemd.
-    type: string
   MongoDbLoggingSource:
     type: json
     description: Fluentd logging configuration for mongodb.
@@ -44,6 +40,10 @@ parameters:
       format: >-
         /(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+\+\d{4})
         (?<message>.*)$/
+  MongodbMemoryLimit:
+    default: '20G'
+    description: Limit the amount of memory mongodb uses with systemd.
+    type: string
   EnableInternalTLS:
     type: boolean
     default: false
@@ -73,9 +73,6 @@ outputs:
     description: Service mongodb using composable services.
     value:
       service_name: mongodb
-      logging_groups:
-        - mongodb
-      logging_source: {get_param: MongoDbLoggingSource}
       config_settings:
         map_merge:
           - get_attr: [MongoDbBase, role_data, config_settings]
@@ -123,6 +120,12 @@ outputs:
                       params:
                         NETWORK: {get_param: [ServiceNetMap, MongodbNetwork]}
               - {}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_mongodb:
+            - mongodb
+          tripleo_fluentd_sources_mongodb:
+            - {get_param: MongoDbLoggingSource}
       step_config: |
         include ::tripleo::profile::base::database::mongodb
       upgrade_tasks:
diff --git a/puppet/services/ec2-api.yaml b/puppet/services/ec2-api.yaml
index 36bb6cf7c6..7ee1b5fee0 100644
--- a/puppet/services/ec2-api.yaml
+++ b/puppet/services/ec2-api.yaml
@@ -96,9 +96,6 @@ outputs:
     value:
       service_name: ec2_api
       monitoring_subscription: {get_param: MonitoringSubscriptionEc2Api}
-      logging_source: {get_param: Ec2ApiLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
         - get_attr: [TLSProxyBase, role_data, config_settings]
@@ -179,6 +176,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::nova::ec2api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_ec2_api:
+            - nova
+          tripleo_fluentd_sources_ec2_api:
+            - {get_param: Ec2ApiLoggingSource}
         keystone:
           ec2api::keystone::auth::tenant: 'service'
           ec2api::keystone::auth::public_url: {get_param: [EndpointMap, Ec2ApiPublic, uri]}
diff --git a/puppet/services/glance-api.yaml b/puppet/services/glance-api.yaml
index 1f6423edf0..15991ddae7 100644
--- a/puppet/services/glance-api.yaml
+++ b/puppet/services/glance-api.yaml
@@ -179,9 +179,6 @@ outputs:
     value:
       service_name: glance_api
       monitoring_subscription: {get_param: MonitoringSubscriptionGlanceApi}
-      logging_source: {get_param: GlanceApiLoggingSource}
-      logging_groups:
-        - glance
       config_settings:
         map_merge:
           - get_attr: [TLSProxyBase, role_data, config_settings]
@@ -268,6 +265,11 @@ outputs:
             - {}
             - glance::api::workers: {get_param: GlanceWorkers}
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_glance_api:
+            - glance
+          tripleo_fluentd_sources_glance_api:
+            - {get_param: GlanceApiLoggingSource}
         keystone:
           glance::keystone::auth::public_url: {get_param: [EndpointMap, GlancePublic, uri]}
           glance::keystone::auth::internal_url: {get_param: [EndpointMap, GlanceInternal, uri]}
diff --git a/puppet/services/gnocchi-api.yaml b/puppet/services/gnocchi-api.yaml
index 285a347fbb..bb41974c3e 100644
--- a/puppet/services/gnocchi-api.yaml
+++ b/puppet/services/gnocchi-api.yaml
@@ -92,9 +92,6 @@ outputs:
     value:
       service_name: gnocchi_api
       monitoring_subscription: {get_param: MonitoringSubscriptionGnocchiApi}
-      logging_source: {get_param: GnocchiApiLoggingSource}
-      logging_groups:
-        - gnocchi
       config_settings:
         map_merge:
           - get_attr: [ApacheServiceBase, role_data, config_settings]
@@ -133,6 +130,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::gnocchi::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_gnocchi_api:
+            - gnocchi
+          tripleo_fluentd_sources_gnocchi_api:
+            - {get_param: GnocchiApiLoggingSource}
         keystone:
           gnocchi::keystone::auth::admin_url: { get_param: [ EndpointMap, GnocchiAdmin, uri ] }
           gnocchi::keystone::auth::internal_url: {get_param: [EndpointMap, GnocchiInternal, uri]}
diff --git a/puppet/services/heat-api-cfn.yaml b/puppet/services/heat-api-cfn.yaml
index 7d0f39405d..8e9e206148 100644
--- a/puppet/services/heat-api-cfn.yaml
+++ b/puppet/services/heat-api-cfn.yaml
@@ -86,9 +86,6 @@ outputs:
     value:
       service_name: heat_api_cfn
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatApiCnf}
-      logging_source: {get_param: HeatApiCfnLoggingSource}
-      logging_groups:
-        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]
@@ -121,6 +118,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::heat::api_cfn
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_heat_api_cfn:
+            - heat
+          tripleo_fluentd_sources_heat_api_cfn:
+            - {get_param: HeatApiCfnLoggingSource}
         keystone:
           map_merge:
             - get_attr: [HeatBase, role_data, service_config_settings, keystone]
diff --git a/puppet/services/heat-api.yaml b/puppet/services/heat-api.yaml
index 6d33d9b5c8..21c8094edd 100644
--- a/puppet/services/heat-api.yaml
+++ b/puppet/services/heat-api.yaml
@@ -92,9 +92,6 @@ outputs:
     value:
       service_name: heat_api
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatApi}
-      logging_source: {get_param: HeatApiLoggingSource}
-      logging_groups:
-        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]
@@ -128,6 +125,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::heat::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_heat_api:
+            - heat
+          tripleo_fluentd_sources_heat_api:
+            - {get_param: HeatApiLoggingSource}
         keystone:
           map_merge:
             - get_attr: [HeatBase, role_data, service_config_settings, keystone]
diff --git a/puppet/services/heat-engine.yaml b/puppet/services/heat-engine.yaml
index 108936574d..4ddbdd7452 100644
--- a/puppet/services/heat-engine.yaml
+++ b/puppet/services/heat-engine.yaml
@@ -89,9 +89,6 @@ outputs:
     value:
       service_name: heat_engine
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatEngine}
-      logging_source: {get_param: HeatEngineLoggingSource}
-      logging_groups:
-        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]
@@ -144,6 +141,11 @@ outputs:
         include ::tripleo::profile::base::heat::engine
 
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_heat_engine:
+            - heat
+          tripleo_fluentd_sources_heat_engine:
+            - {get_param: HeatEngineLoggingSource}
         mysql:
           heat::db::mysql::password: {get_param: HeatPassword}
           heat::db::mysql::user: heat
diff --git a/puppet/services/keystone.yaml b/puppet/services/keystone.yaml
index c112cce867..83e54a13fc 100644
--- a/puppet/services/keystone.yaml
+++ b/puppet/services/keystone.yaml
@@ -325,9 +325,6 @@ outputs:
     value:
       service_name: keystone
       monitoring_subscription: {get_param: MonitoringSubscriptionKeystone}
-      logging_source: {get_param: KeystoneLoggingSource}
-      logging_groups:
-        - keystone
       config_settings:
         map_merge:
           - get_attr: [ApacheServiceBase, role_data, config_settings]
@@ -503,6 +500,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::keystone
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_keystone:
+            - keystone
+          tripleo_fluentd_sources_keystone:
+            - {get_param: KeystoneLoggingSource}
         mysql:
           keystone::db::mysql::password: {get_param: AdminToken}
           keystone::db::mysql::user: keystone
diff --git a/puppet/services/neutron-api.yaml b/puppet/services/neutron-api.yaml
index de0c127720..403d9984d9 100644
--- a/puppet/services/neutron-api.yaml
+++ b/puppet/services/neutron-api.yaml
@@ -135,9 +135,6 @@ outputs:
     value:
       service_name: neutron_api
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronServer}
-      logging_source: {get_param: NeutronApiLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -205,6 +202,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::neutron::server
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_api:
+            - neutron
+          tripleo_fluentd_sources_neutron_api:
+            - {get_param: NeutronApiLoggingSource}
         keystone:
           neutron::keystone::auth::tenant: 'service'
           neutron::keystone::auth::public_url: {get_param: [EndpointMap, NeutronPublic, uri]}
diff --git a/puppet/services/neutron-dhcp.yaml b/puppet/services/neutron-dhcp.yaml
index 871501b252..07dc62afcf 100644
--- a/puppet/services/neutron-dhcp.yaml
+++ b/puppet/services/neutron-dhcp.yaml
@@ -91,9 +91,6 @@ outputs:
     value:
       service_name: neutron_dhcp
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronDhcp}
-      logging_source: {get_param: NeutronDhcpAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -115,6 +112,12 @@ outputs:
                 proto: 'udp'
                 chain: 'OUTPUT'
                 dport: 68
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_dhcp:
+            - neutron
+          tripleo_fluentd_sources_neutron_dhcp:
+            - {get_param: NeutronDhcpAgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::dhcp
       upgrade_tasks:
diff --git a/puppet/services/neutron-l2gw-agent.yaml b/puppet/services/neutron-l2gw-agent.yaml
index ff49bb1e03..3b31fcbc12 100644
--- a/puppet/services/neutron-l2gw-agent.yaml
+++ b/puppet/services/neutron-l2gw-agent.yaml
@@ -83,9 +83,6 @@ outputs:
     value:
       service_name: neutron_l2gw_agent
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronL2gwAgent}
-      logging_source: {get_param: NeutronL2gwAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - neutron::agents::l2gw::ovsdb_hosts: {get_param: L2gwAgentOvsdbHosts}
@@ -108,6 +105,12 @@ outputs:
                     dport: {get_param: L2gwAgentManagerTableListeningPort}
               - null
 
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_l2gw_agent:
+            - neutron
+          tripleo_fluentd_sources_neutron_l2gw_agent:
+            - {get_param: NeutronL2gwAgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::agents::l2gw
       upgrade_tasks:
diff --git a/puppet/services/neutron-l3-compute-dvr.yaml b/puppet/services/neutron-l3-compute-dvr.yaml
index d526977031..ae31e677d5 100644
--- a/puppet/services/neutron-l3-compute-dvr.yaml
+++ b/puppet/services/neutron-l3-compute-dvr.yaml
@@ -75,9 +75,6 @@ outputs:
     value:
       service_name: neutron_l3_compute_dvr
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronL3Dvr}
-      logging_source: {get_param: NeutronL3ComputeAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -87,5 +84,11 @@ outputs:
             - external_network_bridge_empty
             - {}
             - neutron::agents::l3::external_network_bridge: {get_param: NeutronExternalNetworkBridge}
+        service_config_settings:
+          fluentd:
+            tripleo_fluentd_groups_neutron_l3_compute_dvr:
+              - neutron
+            tripleo_fluentd_sources_neutron_l3_compute_dvr:
+              - {get_param: NeutronL3ComputeAgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::l3
diff --git a/puppet/services/neutron-l3.yaml b/puppet/services/neutron-l3.yaml
index 1b9ca2f889..fec943ba11 100644
--- a/puppet/services/neutron-l3.yaml
+++ b/puppet/services/neutron-l3.yaml
@@ -89,9 +89,6 @@ outputs:
     value:
       service_name: neutron_l3
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronL3}
-      logging_source: {get_param: NeutronL3AgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
         - get_attr: [NeutronBase, role_data, config_settings]
@@ -109,6 +106,12 @@ outputs:
           - external_network_bridge_empty
           - {}
           - neutron::agents::l3::external_network_bridge: {get_param: NeutronExternalNetworkBridge}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_l3:
+            - neutron
+          tripleo_fluentd_sources_neutron_l3:
+            - {get_param: NeutronL3AgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::l3
       upgrade_tasks:
diff --git a/puppet/services/neutron-metadata.yaml b/puppet/services/neutron-metadata.yaml
index 02643fadde..983f1521ac 100644
--- a/puppet/services/neutron-metadata.yaml
+++ b/puppet/services/neutron-metadata.yaml
@@ -94,9 +94,6 @@ outputs:
     value:
       service_name: neutron_metadata
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronMetadata}
-      logging_source: {get_param: NeutronMetadataAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -126,6 +123,12 @@ outputs:
             - neutron_workers_unset
             - {}
             - neutron::agents::metadata::metadata_workers: {get_param: NeutronWorkers}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_metadata:
+            - neutron
+          tripleo_fluentd_sources_neutron_metadata:
+            - {get_param: NeutronMetadataAgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::metadata
       upgrade_tasks:
diff --git a/puppet/services/neutron-ovs-agent.yaml b/puppet/services/neutron-ovs-agent.yaml
index 69530c074a..9d47583303 100644
--- a/puppet/services/neutron-ovs-agent.yaml
+++ b/puppet/services/neutron-ovs-agent.yaml
@@ -128,9 +128,6 @@ outputs:
     value:
       service_name: neutron_ovs_agent
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronOvs}
-      logging_source: {get_param: NeutronOpenVswitchAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -158,6 +155,12 @@ outputs:
             - no_firewall_driver
             - {}
             - neutron::agents::ml2::ovs::firewall_driver: {get_param: NeutronOVSFirewallDriver}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_ovs_agent:
+            - neutron
+          tripleo_fluentd_sources_neutron_ovs_agent:
+            - {get_param: NeutronOpenVswitchAgentLoggingSource}
       step_config: |
         include ::tripleo::profile::base::neutron::ovs
       upgrade_tasks:
diff --git a/puppet/services/nova-api.yaml b/puppet/services/nova-api.yaml
index 98584c5f0e..24ab672fea 100644
--- a/puppet/services/nova-api.yaml
+++ b/puppet/services/nova-api.yaml
@@ -111,9 +111,6 @@ outputs:
     value:
       service_name: nova_api
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaApi}
-      logging_source: {get_param: NovaApiLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
         - get_attr: [NovaBase, role_data, config_settings]
@@ -170,6 +167,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::nova::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_api:
+            - nova
+          tripleo_fluentd_sources_nova_api:
+            - {get_param: NovaApiLoggingSource}
         mysql:
           map_merge:
           - {get_attr: [NovaBase, role_data, service_config_settings, mysql]}
diff --git a/puppet/services/nova-compute.yaml b/puppet/services/nova-compute.yaml
index 53f92e1a1c..607401495e 100644
--- a/puppet/services/nova-compute.yaml
+++ b/puppet/services/nova-compute.yaml
@@ -153,9 +153,6 @@ outputs:
     value:
       service_name: nova_compute
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaCompute}
-      logging_source: {get_param: NovaComputeLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]
@@ -217,6 +214,11 @@ outputs:
         # Until then, just include it in the default nova-compute role.
         include tripleo::profile::base::nova::compute::libvirt
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_compute:
+            - nova
+          tripleo_fluentd_sources_nova_compute:
+            - {get_param: NovaComputeLoggingSource}
         collectd:
           tripleo.collectd.plugins.nova_compute:
             - virt
diff --git a/puppet/services/nova-conductor.yaml b/puppet/services/nova-conductor.yaml
index 652ef70d67..53f086b84c 100644
--- a/puppet/services/nova-conductor.yaml
+++ b/puppet/services/nova-conductor.yaml
@@ -34,14 +34,14 @@ parameters:
     default: 0
     description: Number of workers for Nova services.
     type: number
-  MonitoringSubscriptionNovaConductor:
-    default: 'overcloud-nova-conductor'
-    type: string
   NovaConductorLoggingSource:
     type: json
     default:
       tag: openstack.nova.conductor
       path: /var/log/nova/nova-conductor.log
+  MonitoringSubscriptionNovaConductor:
+    default: 'overcloud-nova-conductor'
+    type: string
   UpgradeLevelNovaCompute:
     type: string
     description: Nova Compute upgrade level
@@ -67,9 +67,6 @@ outputs:
     value:
       service_name: nova_conductor
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaConductor}
-      logging_source: {get_param: NovaConductorLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
         - get_attr: [NovaBase, role_data, config_settings]
@@ -78,6 +75,12 @@ outputs:
           - nova_workers_zero
           - {}
           - nova::conductor::workers: {get_param: NovaWorkers}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_conductor:
+            - nova
+          tripleo_fluentd_sources_nova_conductor:
+            - {get_param: NovaConductorLoggingSource}
       step_config: |
         include tripleo::profile::base::nova::conductor
       upgrade_tasks:
diff --git a/puppet/services/nova-consoleauth.yaml b/puppet/services/nova-consoleauth.yaml
index 0c8844db55..889db8434d 100644
--- a/puppet/services/nova-consoleauth.yaml
+++ b/puppet/services/nova-consoleauth.yaml
@@ -56,11 +56,14 @@ outputs:
     value:
       service_name: nova_consoleauth
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaConsoleauth}
-      logging_source: {get_param: NovaConsoleauthLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         get_attr: [NovaBase, role_data, config_settings]
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_consoleauth:
+            - nova
+          tripleo_fluentd_sources_nova_consoleauth:
+            - {get_param: NovaConsoleauthLoggingSource}
       step_config: |
         include tripleo::profile::base::nova::consoleauth
       upgrade_tasks:
diff --git a/puppet/services/nova-placement.yaml b/puppet/services/nova-placement.yaml
index 4bb0a671a3..b5f4b31f93 100644
--- a/puppet/services/nova-placement.yaml
+++ b/puppet/services/nova-placement.yaml
@@ -85,9 +85,6 @@ outputs:
     value:
       service_name: nova_placement
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaPlacement}
-      logging_source: {get_param: NovaPlacementLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
         - get_attr: [NovaBase, role_data, config_settings]
@@ -123,6 +120,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::nova::placement
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_placement:
+            - nova
+          tripleo_fluentd_sources_nova_placement:
+            - {get_param: NovaPlacementLoggingSource}
         keystone:
           nova::keystone::auth_placement::tenant: 'service'
           nova::keystone::auth_placement::public_url: {get_param: [EndpointMap, NovaPlacementPublic, uri]}
diff --git a/puppet/services/nova-scheduler.yaml b/puppet/services/nova-scheduler.yaml
index 81a7728621..b861fc85e1 100644
--- a/puppet/services/nova-scheduler.yaml
+++ b/puppet/services/nova-scheduler.yaml
@@ -83,9 +83,6 @@ outputs:
     value:
       service_name: nova_scheduler
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaScheduler}
-      logging_source: {get_param: NovaSchedulerLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]
@@ -94,6 +91,12 @@ outputs:
             nova::scheduler::filter::scheduler_default_filters: {get_param: NovaSchedulerDefaultFilters}
             nova::scheduler::filter::scheduler_max_attempts: {get_param: NovaSchedulerMaxAttempts}
             nova::scheduler::discover_hosts_in_cells_interval: {get_param: NovaSchedulerDiscoverHostsInCellsInterval}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_scheduler:
+            - nova
+          tripleo_fluentd_sources_nova_scheduler:
+            - {get_param: NovaSchedulerLoggingSource}
       step_config: |
         include tripleo::profile::base::nova::scheduler
       upgrade_tasks:
diff --git a/puppet/services/nova-vnc-proxy.yaml b/puppet/services/nova-vnc-proxy.yaml
index 5dd6864cc6..abd00fb834 100644
--- a/puppet/services/nova-vnc-proxy.yaml
+++ b/puppet/services/nova-vnc-proxy.yaml
@@ -56,9 +56,6 @@ outputs:
     value:
       service_name: nova_vnc_proxy
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaVNCProxy}
-      logging_source: {get_param: NovaVncproxyLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]
@@ -77,6 +74,12 @@ outputs:
                 dport:
                   - 6080
                   - 13080
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_vnc_proxy:
+            - nova
+          tripleo_fluentd_sources_nova_vnc_proxy:
+            - {get_param: NovaVncproxyLoggingSource}
       step_config: |
         include tripleo::profile::base::nova::vncproxy
       upgrade_tasks:
diff --git a/puppet/services/octavia-api.yaml b/puppet/services/octavia-api.yaml
index ad59566e20..9ecb39fc21 100644
--- a/puppet/services/octavia-api.yaml
+++ b/puppet/services/octavia-api.yaml
@@ -71,9 +71,6 @@ outputs:
     value:
       service_name: octavia_api
       monitoring_subscription: {get_param: MonitoringSubscriptionOctaviaApi}
-      logging_source: {get_param: OctaviaApiLoggingSource}
-      logging_groups:
-        - octavia
       config_settings:
         map_merge:
           - get_attr: [OctaviaBase, role_data, config_settings]
@@ -102,6 +99,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::octavia::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_octavia_api:
+            - octavia
+          tripleo_fluentd_sources_octavia_api:
+            - {get_param: OctaviaApiLoggingSource}
         keystone:
           octavia::keystone::auth::tenant: 'service'
           octavia::keystone::auth::public_url: {get_param: [EndpointMap, OctaviaPublic, uri]}
diff --git a/puppet/services/octavia-health-manager.yaml b/puppet/services/octavia-health-manager.yaml
index c3d1c1db52..35874394bf 100644
--- a/puppet/services/octavia-health-manager.yaml
+++ b/puppet/services/octavia-health-manager.yaml
@@ -61,14 +61,17 @@ outputs:
     value:
       service_name: octavia_health_manager
       monitoring_subscription: {get_param: MonitoringSubscriptionOctaviaHealthManager}
-      logging_source: {get_param: OctaviaHealthManagerLoggingSource}
-      logging_groups:
-        - octavia
       config_settings:
         map_merge:
           - get_attr: [OctaviaBase, role_data, config_settings]
           - octavia::health_manager::heartbeat_key: {get_param: OctaviaHeartbeatKey}
             octavia::health_manager::event_streamer_driver: 'queue_event_streamer'
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_octavia_health_manager:
+            - octavia
+          tripleo_fluentd_sources_octavia_health_manager:
+            - {get_param: OctaviaHealthManagerLoggingSource}
       step_config: |
         include tripleo::profile::base::octavia::health_manager
 
diff --git a/puppet/services/octavia-housekeeping.yaml b/puppet/services/octavia-housekeeping.yaml
index 656ba1bf0f..de09624d62 100644
--- a/puppet/services/octavia-housekeeping.yaml
+++ b/puppet/services/octavia-housekeeping.yaml
@@ -68,9 +68,6 @@ outputs:
     value:
       service_name: octavia_housekeeping
       monitoring_subscription: {get_param: MonitoringSubscriptionOctaviaHousekeeping}
-      logging_source: {get_param: OctaviaHousekeepingLoggingSource}
-      logging_groups:
-        - octavia
       config_settings:
         map_merge:
           - get_attr: [OctaviaBase, role_data, config_settings]
@@ -79,6 +76,12 @@ outputs:
             - amphora_expiry_is_zero
             - {}
             - octavia::worker::amphora_expiry_age: {get_param: OctaviaAmphoraExpiryAge}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_octavia_housekeeping:
+            - octavia
+          tripleo_fluentd_sources_octavia_housekeeping:
+            - {get_param: OctaviaHousekeepingLoggingSource}
       step_config: |
         include tripleo::profile::base::octavia::housekeeping
 
diff --git a/puppet/services/octavia-worker.yaml b/puppet/services/octavia-worker.yaml
index 37471dd26f..1f701221d9 100644
--- a/puppet/services/octavia-worker.yaml
+++ b/puppet/services/octavia-worker.yaml
@@ -99,9 +99,6 @@ outputs:
     value:
       service_name: octavia_worker
       monitoring_subscription: {get_param: MonitoringSubscriptionOctaviaWorker}
-      logging_source: {get_param: OctaviaWorkerLoggingSource}
-      logging_groups:
-        -octavia
       config_settings:
         map_merge:
           - get_attr: [OctaviaBase, role_data, config_settings]
@@ -125,6 +122,12 @@ outputs:
             - {}
             - octavia::certificates::client_cert_data: {get_param: OctaviaClientCert}
 
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_octavia_worker:
+            - octavia
+          tripleo_fluentd_sources_octavia_worker:
+            - {get_param: OctaviaWorkerLoggingSource}
       step_config: |
         include tripleo::profile::base::octavia::worker
 
diff --git a/puppet/services/pacemaker.yaml b/puppet/services/pacemaker.yaml
index b687fa80ae..c45cb3f7de 100644
--- a/puppet/services/pacemaker.yaml
+++ b/puppet/services/pacemaker.yaml
@@ -99,7 +99,6 @@ parameters:
         \[(?<pid>[^ ]*)\]
         (?<host>[^ ]*)
         (?<message>.*)$/
-
   EnableLoadBalancer:
     default: true
     description: Whether to deploy a LoadBalancer on the Controller
@@ -111,9 +110,6 @@ outputs:
     value:
       service_name: pacemaker
       monitoring_subscription: {get_param: MonitoringSubscriptionPacemaker}
-      logging_groups:
-        - haclient
-      logging_source: {get_param: PacemakerLoggingSource}
       config_settings:
         pacemaker::corosync::cluster_name: 'tripleo_cluster'
         pacemaker::corosync::manage_fw: false
@@ -142,6 +138,12 @@ outputs:
                 - {get_param: PcsdPassword}
                 - {get_param: [DefaultPasswords, pcsd_password]}
         tripleo::profile::base::pacemaker::remote_authkey: {get_param: PacemakerRemoteAuthkey}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_pacemaker:
+            - haclient
+          tripleo_fluentd_sources_pacemaker:
+            - {get_param: PacemakerLoggingSource}
       step_config: |
         include ::tripleo::profile::base::pacemaker
       upgrade_tasks:
diff --git a/puppet/services/pacemaker/cinder-volume.yaml b/puppet/services/pacemaker/cinder-volume.yaml
index 9fde5d73fa..6440cbf097 100644
--- a/puppet/services/pacemaker/cinder-volume.yaml
+++ b/puppet/services/pacemaker/cinder-volume.yaml
@@ -49,8 +49,6 @@ outputs:
     value:
       service_name: cinder_volume
       monitoring_subscription: {get_attr: [CinderVolumeBase, role_data, monitoring_subscription]}
-      logging_source: {get_attr: [CinderVolumeBase, role_data, logging_source]}
-      logging_groups: {get_attr: [CinderVolumeBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [CinderVolumeBase, role_data, config_settings]
diff --git a/puppet/services/pacemaker_remote.yaml b/puppet/services/pacemaker_remote.yaml
index c9b5025fd8..79e43816a3 100644
--- a/puppet/services/pacemaker_remote.yaml
+++ b/puppet/services/pacemaker_remote.yaml
@@ -96,9 +96,6 @@ outputs:
     value:
       service_name: pacemaker_remote
       monitoring_subscription: {get_param: MonitoringSubscriptionPacemakerRemote}
-      logging_groups:
-        - haclient
-      logging_source: {get_param: PacemakerRemoteLoggingSource}
       config_settings:
         tripleo.pacemaker_remote.firewall_rules:
           '130 pacemaker_remote tcp':
@@ -116,5 +113,11 @@ outputs:
               passwords:
                 - {get_param: PcsdPassword}
                 - {get_param: [DefaultPasswords, pcsd_password]}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_pacemaker_remote:
+            - haclient
+          tripleo_fluentd_sources_pacemaker_remote:
+            - {get_param: PacemakerRemoteLoggingSource}
       step_config: |
         include ::tripleo::profile::base::pacemaker_remote
diff --git a/puppet/services/sahara-api.yaml b/puppet/services/sahara-api.yaml
index f41fbc802f..115d13f71f 100644
--- a/puppet/services/sahara-api.yaml
+++ b/puppet/services/sahara-api.yaml
@@ -74,9 +74,6 @@ outputs:
     value:
       service_name: sahara_api
       monitoring_subscription: {get_param: MonitoringSubscriptionSaharaApi}
-      logging_source: {get_param: SaharaApiLoggingSource}
-      logging_groups:
-        - sahara
       config_settings:
         map_merge:
           - get_attr: [SaharaBase, role_data, config_settings]
@@ -94,6 +91,12 @@ outputs:
                 dport:
                   - 8386
                   - 13386
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_sahara_api:
+            - sahara
+          tripleo_fluentd_sources_sahara_api:
+            - {get_param: SaharaApiLoggingSource}
       step_config: |
         include ::tripleo::profile::base::sahara::api
       service_config_settings:
diff --git a/puppet/services/sahara-engine.yaml b/puppet/services/sahara-engine.yaml
index b419a9f3e0..184aaf7a70 100644
--- a/puppet/services/sahara-engine.yaml
+++ b/puppet/services/sahara-engine.yaml
@@ -56,12 +56,15 @@ outputs:
     value:
       service_name: sahara_engine
       monitoring_subscription: {get_param: MonitoringSubscriptionSaharaEngine}
-      logging_source: {get_param: SaharaEngineLoggingSource}
-      logging_groups:
-        - sahara
       config_settings:
         map_merge:
           - get_attr: [SaharaBase, role_data, config_settings]
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_sahara_engine:
+            - sahara
+          tripleo_fluentd_sources_sahara_engine:
+            - {get_param: SaharaEngineLoggingSource}
       step_config: |
         include ::tripleo::profile::base::sahara::engine
       upgrade_tasks: