From 0d9298bb8fc9903ff42e3d2402cd44e765f4c822 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 9 Aug 2016 16:20:18 -0400 Subject: [PATCH] Add fluentd client service This implements support for installing fluentd agents as a composable service on the overcloud. Depends-On: I2e1abe4d8c8359e56ff626255ee50c9cacca1940 Implements: tripleo-opstools-centralized-logging Change-Id: I23b0e23881b742158fcfb6b8c145a3211d45086e --- capabilities-map.yaml | 8 + environments/logging-environment.yaml | 29 ++++ overcloud-resource-registry-puppet.yaml | 2 + overcloud.j2.yaml | 2 + puppet/ceph-storage.yaml | 8 + puppet/cinder-storage.yaml | 8 + puppet/compute.yaml | 8 + puppet/controller.yaml | 8 + puppet/services/ceilometer-agent-central.yaml | 8 + .../ceilometer-agent-notification.yaml | 8 + puppet/services/ceilometer-api.yaml | 8 + puppet/services/ceilometer-collector.yaml | 8 + puppet/services/cinder-api.yaml | 8 + puppet/services/cinder-scheduler.yaml | 8 + puppet/services/cinder-volume.yaml | 8 + puppet/services/database/mongodb.yaml | 12 ++ puppet/services/glance-api.yaml | 8 + puppet/services/glance-registry.yaml | 8 + puppet/services/gnocchi-api.yaml | 8 + puppet/services/heat-api-cfn.yaml | 8 + puppet/services/heat-api-cloudwatch.yaml | 8 + puppet/services/heat-api.yaml | 8 + puppet/services/heat-engine.yaml | 8 + puppet/services/keystone.yaml | 8 + puppet/services/logging/fluentd-base.yaml | 36 ++++ puppet/services/logging/fluentd-client.yaml | 64 ++++++++ puppet/services/logging/fluentd-config.yaml | 154 ++++++++++++++++++ puppet/services/neutron-api.yaml | 8 + puppet/services/neutron-dhcp.yaml | 8 + puppet/services/neutron-l3-compute-dvr.yaml | 8 + puppet/services/neutron-l3.yaml | 8 + puppet/services/neutron-metadata.yaml | 8 + puppet/services/neutron-ovs-agent.yaml | 8 + puppet/services/nova-api.yaml | 8 + puppet/services/nova-compute.yaml | 8 + puppet/services/nova-conductor.yaml | 8 + puppet/services/nova-consoleauth.yaml | 8 + puppet/services/nova-scheduler.yaml | 8 + puppet/services/nova-vnc-proxy.yaml | 8 + puppet/services/pacemaker.yaml | 13 ++ puppet/services/pacemaker/cinder-api.yaml | 2 + .../services/pacemaker/cinder-scheduler.yaml | 2 + puppet/services/pacemaker/cinder-volume.yaml | 2 + puppet/services/pacemaker/glance-api.yaml | 2 + .../services/pacemaker/glance-registry.yaml | 2 + puppet/services/pacemaker/heat-api-cfn.yaml | 2 + .../pacemaker/heat-api-cloudwatch.yaml | 2 + puppet/services/pacemaker/heat-api.yaml | 2 + puppet/services/pacemaker/heat-engine.yaml | 2 + puppet/services/pacemaker/keystone.yaml | 2 + puppet/services/pacemaker/neutron-dhcp.yaml | 2 + puppet/services/pacemaker/neutron-l3.yaml | 2 + .../services/pacemaker/neutron-metadata.yaml | 2 + .../services/pacemaker/neutron-ovs-agent.yaml | 2 + puppet/services/pacemaker/nova-api.yaml | 2 + puppet/services/pacemaker/nova-conductor.yaml | 2 + .../services/pacemaker/nova-consoleauth.yaml | 2 + puppet/services/pacemaker/nova-scheduler.yaml | 2 + puppet/services/pacemaker/nova-vnc-proxy.yaml | 2 + puppet/services/pacemaker/sahara-api.yaml | 2 + puppet/services/pacemaker/sahara-engine.yaml | 2 + puppet/services/sahara-api.yaml | 8 + puppet/services/sahara-engine.yaml | 8 + puppet/services/services.yaml | 39 +++++ puppet/swift-storage.yaml | 8 + roles_data.yaml | 5 + 66 files changed, 678 insertions(+) create mode 100644 environments/logging-environment.yaml create mode 100644 puppet/services/logging/fluentd-base.yaml create mode 100644 puppet/services/logging/fluentd-client.yaml create mode 100644 puppet/services/logging/fluentd-config.yaml diff --git a/capabilities-map.yaml b/capabilities-map.yaml index 962dfb99ab..14d50f1909 100644 --- a/capabilities-map.yaml +++ b/capabilities-map.yaml @@ -236,3 +236,11 @@ topics: description: requires: - overcloud-resource-registry-puppet.yaml + - title: Centralized logging support + description: Enable centralized logging clients (fluentd) + environments: + - file: environments/logging-environment.yaml + title: Enable fluentd client + description: + requires: + - overcloud-resource-registry-puppet.yaml diff --git a/environments/logging-environment.yaml b/environments/logging-environment.yaml new file mode 100644 index 0000000000..eefa7026bf --- /dev/null +++ b/environments/logging-environment.yaml @@ -0,0 +1,29 @@ +## A Heat environment file which can be used to set up +## logging agents + +resource_registry: + OS::TripleO::Services::FluentdClient: ../puppet/services/logging/fluentd-client.yaml + +parameter_defaults: + +## Simple configuration +# +# LoggingServers: +# - host: log0.example.com +# port: 24224 +# - host: log1.example.com +# port: 24224 +# +## Example SSL configuration +## (note the use of port 24284 for ssl connections) +# +# LoggingServers: +# - host: 192.0.2.11 +# port: 24284 +# LoggingUsesSSL: true +# LoggingSharedKey: secret +# LoggingSSLCertificate: | +# -----BEGIN CERTIFICATE----- +# ...certificate data here... +# -----END CERTIFICATE----- + diff --git a/overcloud-resource-registry-puppet.yaml b/overcloud-resource-registry-puppet.yaml index d5fdaa0912..a15e08abca 100644 --- a/overcloud-resource-registry-puppet.yaml +++ b/overcloud-resource-registry-puppet.yaml @@ -212,6 +212,8 @@ resource_registry: OS::TripleO::Services::GnocchiStatsd: puppet/services/gnocchi-statsd.yaml OS::TripleO::Services::VipHosts: puppet/services/vip-hosts.yaml # Services that are disabled by default (use relevant environment files): + OS::TripleO::Services::FluentdClient: OS::Heat::None + OS::TripleO::LoggingConfiguration: puppet/services/logging/fluentd-config.yaml OS::Tripleo::Services::ManilaApi: OS::Heat::None OS::Tripleo::Services::ManilaScheduler: OS::Heat::None OS::Tripleo::Services::ManilaShare: OS::Heat::None diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml index 5c0d91481e..6ca4a14a1a 100644 --- a/overcloud.j2.yaml +++ b/overcloud.j2.yaml @@ -265,6 +265,8 @@ resources: {% endfor %} ServiceNames: {get_attr: [{{role.name}}ServiceChain, role_data, service_names]} MonitoringSubscriptions: {get_attr: [{{role.name}}ServiceChain, role_data, monitoring_subscriptions]} + LoggingSources: {get_attr: [{{role.name}}ServiceChain, role_data, logging_sources]} + LoggingGroups: {get_attr: [{{role.name}}ServiceChain, role_data, logging_groups]} {% endfor %} allNodesConfig: diff --git a/puppet/ceph-storage.yaml b/puppet/ceph-storage.yaml index 62748f94ce..03a53b0088 100644 --- a/puppet/ceph-storage.yaml +++ b/puppet/ceph-storage.yaml @@ -102,6 +102,12 @@ parameters: type: string description: Command which will be run whenever configuration data changes default: os-refresh-config --timeout 14400 + LoggingSources: + type: json + default: [] + LoggingGroups: + type: comma_delimited_list + default: [] resources: CephStorage: @@ -275,6 +281,8 @@ resources: ceph: mapped_data: tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources} + tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups} # Resource for site-specific injection of root certificate NodeTLSCAData: diff --git a/puppet/cinder-storage.yaml b/puppet/cinder-storage.yaml index f5118c2cfa..a66ea08b21 100644 --- a/puppet/cinder-storage.yaml +++ b/puppet/cinder-storage.yaml @@ -96,6 +96,12 @@ parameters: type: string description: Command which will be run whenever configuration data changes default: os-refresh-config --timeout 14400 + LoggingSources: + type: json + default: [] + LoggingGroups: + type: comma_delimited_list + default: [] resources: BlockStorage: @@ -271,6 +277,8 @@ resources: volume: mapped_data: tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources} + tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups} # Resource for site-specific injection of root certificate NodeTLSCAData: diff --git a/puppet/compute.yaml b/puppet/compute.yaml index 05b8d06572..0205d0a64b 100644 --- a/puppet/compute.yaml +++ b/puppet/compute.yaml @@ -114,6 +114,12 @@ parameters: type: string description: Command which will be run whenever configuration data changes default: os-refresh-config --timeout 14400 + LoggingSources: + type: json + default: [] + LoggingGroups: + type: comma_delimited_list + default: [] resources: @@ -289,6 +295,8 @@ resources: compute: mapped_data: tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources} + tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups} NovaComputeDeployment: type: OS::TripleO::SoftwareDeployment diff --git a/puppet/controller.yaml b/puppet/controller.yaml index 33ed51c02c..ccb517f87c 100644 --- a/puppet/controller.yaml +++ b/puppet/controller.yaml @@ -128,6 +128,12 @@ parameters: type: string description: Command which will be run whenever configuration data changes default: os-refresh-config --timeout 14400 + LoggingSources: + type: json + default: [] + LoggingGroups: + type: comma_delimited_list + default: [] parameter_groups: - label: deprecated @@ -349,6 +355,8 @@ resources: # Misc tripleo::haproxy::service_certificate: {get_attr: [NodeTLSData, deployed_ssl_certificate_path]} tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources} + tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups} # Hook for site-specific additional pre-deployment config, e.g extra hieradata ControllerExtraConfigPre: diff --git a/puppet/services/ceilometer-agent-central.yaml b/puppet/services/ceilometer-agent-central.yaml index 5d980d7996..2ae46d0e92 100644 --- a/puppet/services/ceilometer-agent-central.yaml +++ b/puppet/services/ceilometer-agent-central.yaml @@ -25,6 +25,11 @@ parameters: MonitoringSubscriptionCeilometerCentral: default: 'overcloud-ceilometer-agent-central' type: string + CeilometerAgentCentralLoggingSource: + type: json + default: + tag: openstack.ceilometer.agent.central + path: /var/log/ceilometer/central.log resources: CeilometerServiceBase: @@ -40,6 +45,9 @@ outputs: value: service_name: ceilometer_agent_central monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerCentral} + logging_source: {get_param: CeilometerAgentCentralLoggingSource} + logging_groups: + - ceilometer config_settings: map_merge: - get_attr: [CeilometerServiceBase, role_data, config_settings] diff --git a/puppet/services/ceilometer-agent-notification.yaml b/puppet/services/ceilometer-agent-notification.yaml index bedb8b0415..ea403aa15f 100644 --- a/puppet/services/ceilometer-agent-notification.yaml +++ b/puppet/services/ceilometer-agent-notification.yaml @@ -21,6 +21,11 @@ parameters: MonitoringSubscriptionCeilometerNotification: default: 'overcloud-ceilometer-agent-notification' type: string + CeilometerAgentNotificationLoggingSource: + type: json + default: + tag: openstack.ceilometer.agent.notification + path: /var/log/ceilometer/agent-notification.log resources: @@ -37,6 +42,9 @@ outputs: value: service_name: ceilometer_agent_notification monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerNotification} + logging_source: {get_param: CeilometerAgentNotificationLoggingSource} + logging_groups: + - ceilometer config_settings: get_attr: [CeilometerServiceBase, role_data, config_settings] step_config: | diff --git a/puppet/services/ceilometer-api.yaml b/puppet/services/ceilometer-api.yaml index 5df9f2b37f..c6e3650250 100644 --- a/puppet/services/ceilometer-api.yaml +++ b/puppet/services/ceilometer-api.yaml @@ -21,6 +21,11 @@ parameters: MonitoringSubscriptionCeilometerApi: default: 'overcloud-ceilometer-api' type: string + CeilometerApiLoggingSource: + type: json + default: + tag: openstack.ceilometer.api + path: /var/log/ceilometer/api.log resources: @@ -44,6 +49,9 @@ outputs: value: service_name: ceilometer_api monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerApi} + logging_source: {get_param: CeilometerApiLoggingSource} + logging_groups: + - ceilometer config_settings: map_merge: - get_attr: [ApacheServiceBase, role_data, config_settings] diff --git a/puppet/services/ceilometer-collector.yaml b/puppet/services/ceilometer-collector.yaml index 9dbb275984..2fc9aec0ce 100644 --- a/puppet/services/ceilometer-collector.yaml +++ b/puppet/services/ceilometer-collector.yaml @@ -21,6 +21,11 @@ parameters: MonitoringSubscriptionCeilometerCollector: default: 'overcloud-ceilometer-collector' type: string + CeilometerCollectorLoggingSource: + type: json + default: + tag: openstack.ceilometer.collector + path: /var/log/ceilometer/collector.log resources: CeilometerServiceBase: @@ -36,6 +41,9 @@ outputs: value: service_name: ceilometer_collector monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerCollector} + logging_source: {get_param: CeilometerCollectorLoggingSource} + logging_groups: + - ceilometer config_settings: get_attr: [CeilometerServiceBase, role_data, config_settings] step_config: | diff --git a/puppet/services/cinder-api.yaml b/puppet/services/cinder-api.yaml index 94c94a659b..1dae9f15a0 100644 --- a/puppet/services/cinder-api.yaml +++ b/puppet/services/cinder-api.yaml @@ -34,6 +34,11 @@ parameters: MonitoringSubscriptionCinderApi: default: 'overcloud-cinder-api' type: string + CinderApiLoggingSource: + type: json + default: + tag: openstack.cinder.api + path: /var/log/cinder/cinder-api.log resources: @@ -50,6 +55,9 @@ outputs: value: service_name: cinder_api monitoring_subscription: {get_param: MonitoringSubscriptionCinderApi} + logging_source: {get_param: CinderApiLoggingSource} + logging_groups: + - cinder config_settings: map_merge: - get_attr: [CinderBase, role_data, config_settings] diff --git a/puppet/services/cinder-scheduler.yaml b/puppet/services/cinder-scheduler.yaml index 1326e26736..94c263ea55 100644 --- a/puppet/services/cinder-scheduler.yaml +++ b/puppet/services/cinder-scheduler.yaml @@ -21,6 +21,11 @@ parameters: MonitoringSubscriptionCinderScheduler: default: 'overcloud-cinder-scheduler' type: string + CinderSchedulerLoggingSource: + type: json + default: + tag: openstack.cinder.scheduler + path: /var/log/cinder/cinder-scheduler.log resources: @@ -37,6 +42,9 @@ outputs: value: service_name: cinder_scheduler monitoring_subscription: {get_param: MonitoringSubscriptionCinderScheduler} + logging_source: {get_param: CinderSchedulerLoggingSource} + logging_groups: + - cinder config_settings: map_merge: - get_attr: [CinderBase, role_data, config_settings] diff --git a/puppet/services/cinder-volume.yaml b/puppet/services/cinder-volume.yaml index c84c784e43..82e16f3975 100644 --- a/puppet/services/cinder-volume.yaml +++ b/puppet/services/cinder-volume.yaml @@ -59,6 +59,11 @@ parameters: MonitoringSubscriptionCinderVolume: default: 'overcloud-cinder-volume' type: string + CinderVolumeLoggingSource: + type: json + default: + tag: openstack.cinder.volume + path: /var/log/cinder/cinder-volume.log resources: @@ -75,6 +80,9 @@ outputs: value: service_name: cinder_volume monitoring_subscription: {get_param: MonitoringSubscriptionCinderVolume} + logging_source: {get_param: CinderVolumeLoggingSource} + logging_groups: + - cinder config_settings: map_merge: - get_attr: [CinderBase, role_data, config_settings] diff --git a/puppet/services/database/mongodb.yaml b/puppet/services/database/mongodb.yaml index 36962a344b..01daeafe89 100644 --- a/puppet/services/database/mongodb.yaml +++ b/puppet/services/database/mongodb.yaml @@ -19,6 +19,15 @@ parameters: description: Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry. type: json + MongoDbLoggingSource: + type: json + description: Fluentd logging configuration for mongodb. + default: + tag: database.mongodb + path: /var/log/mongodb/mongodb.log + format: >- + /(?