Base profile for rsyslogd

Adds profile for rsyslogd composable service which aims to replace fluentd
with the same behaviour. This means that rsyslog will be tailing OpenStack
log files and forwarding it to central log collector (ELK)

Change-Id: I5e5aae692482ee1de6de298eb18c49563a2d7d29

Puppetfile_extras

@@ -54,6 +54,10 @@ mod 'opendaylight',
   :git => 'https://git.opendaylight.org/gerrit/integration/packaging/puppet-opendaylight',
   :ref => 'master'
 
+mod 'rsyslog',
+  :git => 'https://github.com/voxpupuli/puppet-rsyslog',
+  :ref => 'master'
+
 mod 'ssh',
   :git => 'https://github.com/saz/puppet-ssh',
   :ref => 'v3.0.1'

manifests/profile/base/logging/rsyslog.pp

@@ -0,0 +1,54 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: tripleo::profile::base::logging::rsyslog
+#
+# rsyslogd configuration for TripleO
+#
+# === Parameters
+#
+# [*step*]
+#   (Optional) String. The current step of the deployment
+#   Defaults to hiera('step')
+#
+# [*elasticsearch*]
+#   (Optional) Hash. Configuration for output plugin omelasticsearch.
+#
+# [*service_names*]
+#   (Optional) List of services enabled on the current role. This is used
+#   to obtain per-service configuration information.
+#
+class tripleo::profile::base::logging::rsyslog (
+  $step            = Integer(hiera('step')),
+  $elasticsearch   = undef,
+  $service_names   = hiera('service_names', [])
+) {
+  if $step >= 2 {
+    $modules = {
+      'imfile'          => {},
+      'omelasticsearch' => {},
+    }
+    $actions = {
+      'elasticsearch' => {
+        'type'   => 'omelasticsearch',
+        'config' => $elasticsearch,
+      }
+    }
+    class { '::rsyslog::server':
+      modules => $modules,
+      actions => $actions
+    }
+    tripleo::profile::base::logging::rsyslog::file_input{$service_names: }
+  }
+}

manifests/profile/base/logging/rsyslog/file_input.pp

@@ -0,0 +1,52 @@
+# This is used to look up a list of service-specific rsyslogd configurations
+# in the hiera data provided by THT.
+#
+# [*sources*]
+#   (Optional) List of hashes. Contains configuration of file inputs for given service.
+#   Defaults to hiera("tripleo_logging_sources_${title}", undef)
+#
+# [*default_startmsg*]
+#   (Optional) String. Default POSIX ERE for start of log record. The default enables to forward
+#   multiline tracebacks for most of OpenStack services. It can be overridden either by this
+#   parameter for all file inputs or in each file input separately in THT parameters
+#   tripleo_logging_sources_<service name>.
+#   Defaults to '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) '
+#
+define tripleo::profile::base::logging::rsyslog::file_input (
+  $sources = hiera("tripleo_logging_sources_${title}", undef),
+  $default_startmsg = '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) '
+) {
+  # TODO(mmagr): As a deprecation path for fluentd we gonna use the fluentd THT parameters first.
+  # We can remove below few lines after fluentd is removed and it's template outputs
+  # are renamed to provider-agnostic name in all composable services
+  if !$sources {
+    $fluentd_sources = hiera("tripleo_fluentd_sources_${title}", [])
+    $rsyslog_sources = $fluentd_sources.reduce([]) |$memo, $config| {
+      if $config['startmsg.regex'] {
+        $startmsg = $config['startmsg.regex']
+      } else {
+        $startmsg = $default_startmsg
+      }
+      $memo + [{'file' => $config['path'], 'tag' => $config['tag'], 'startmsg.regex' => $startmsg }]
+    }
+  } else {
+    $rsyslog_sources = $sources.reduce([]) |$memo, $config| {
+      if ! $config['startmsg.regex'] {
+        $record = $config + {'startmsg.regex' => $default_startmsg}
+      } else {
+        $record = $config
+      }
+      $memo + [$record]
+    }
+  }
+
+  $rsyslog_sources.each |$config| {
+    rsyslog::component::input{ "${title}_${config['tag']}":
+      priority => $::rsyslog::input_priority,
+      target   => $::rsyslog::target_file,
+      confdir  => $::rsyslog::confdir,
+      type     => 'imfile',
+      config   => $config
+    }
+  }
+}

releasenotes/notes/rsyslog-205c11903ed92bdf.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Adds profile for rsyslogd composable service which aims to replace fluentd
+    with the same behaviour. This means that rsyslog will be tailing OpenStack
+    log files and forwarding it to central log collector (ELK)

spec/classes/tripleo_profile_base_logging_rsyslog_spec.rb

@@ -0,0 +1,84 @@
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+require 'spec_helper'
+
+horizon_access_log_conf = <<-EOS
+# horizon_openstack.horizon.access
+input(type="imfile"
+  file="/var/log/horizon/access.log"
+  tag="openstack.horizon.access"
+  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
+)
+EOS
+horizon_test_log_conf = <<-EOS
+# horizon_openstack.horizon.test
+input(type="imfile"
+  file="/var/log/horizon/test.log"
+  tag="openstack.horizon.test"
+  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
+)
+EOS
+elastic_conf = <<-EOS
+# elasticsearch
+action(type="omelasticsearch"
+    name="elasticsearch"
+)
+EOS
+
+describe 'tripleo::profile::base::logging::rsyslog' do
+  shared_examples_for 'tripleo::profile::base::logging::rsyslog' do
+    before :each do
+      facts.merge!({ :step => params[:step] })
+    end
+
+    context 'on step 2' do
+      let(:params) { { :step => 2 } }
+
+      it 'should generate a rsyslog config file for horizon from hieradata' do
+        should contain_concat__fragment("rsyslog::component::module::imfile").with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => "module(load=\"imfile\")\n",
+        })
+        should contain_concat__fragment("rsyslog::component::module::omelasticsearch").with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => "module(load=\"omelasticsearch\")\n",
+        })
+        should contain_concat__fragment('rsyslog::component::input::horizon_openstack.horizon.access').with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => horizon_access_log_conf,
+        })
+        should contain_concat__fragment('rsyslog::component::input::horizon_openstack.horizon.test').with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => horizon_test_log_conf,
+        })
+        should contain_concat__fragment("rsyslog::component::action::elasticsearch").with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => elastic_conf,
+        })
+      end
+    end
+  end
+
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let (:facts) {
+        facts
+      }
+      it_behaves_like 'tripleo::profile::base::logging::rsyslog'
+    end
+  end
+end

spec/defines/tripleo_profile_base_logging_rsyslog_file_input_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+foo_log_conf = <<-EOS
+# foobar_foo
+input(type="imfile"
+  file="/path/to/foo.log"
+  tag="foo"
+  startmsg.regex="test"
+)
+EOS
+bar_log_conf = <<-EOS
+# foobar_bar
+input(type="imfile"
+  file="/path/to/bar.log"
+  tag="bar"
+  startmsg.regex="baz"
+)
+EOS
+
+describe 'tripleo::profile::base::logging::rsyslog::file_input' do
+  let(:title) { 'foobar' }
+
+  let :pre_condition do
+    'include ::rsyslog::server'
+  end
+
+  let :params do {
+    'sources' => [
+      {'file' => '/path/to/foo.log', 'tag' => 'foo'},
+      {'file' => '/path/to/bar.log', 'tag' => 'bar', 'startmsg.regex' => 'baz'}
+    ],
+    'default_startmsg' => 'test'
+  }
+  end
+
+  shared_examples_for 'tripleo::profile::base::logging::rsyslog::file_input' do
+    context 'with basic parameters to configure rsyslog file inputs' do
+      it 'should configure the given file inputs' do
+        should contain_concat__fragment('rsyslog::component::input::foobar_foo').with({
+          :target => '/etc/rsyslog.d/50_rsyslog.conf',
+          :content => foo_log_conf,
+        })
+        should contain_concat__fragment('rsyslog::component::input::foobar_bar').with({
+          :target => '/etc/rsyslog.d/50_rsyslog.conf',
+          :content => bar_log_conf,
+        })
+      end
+    end
+  end
+
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let(:facts) do
+        facts.merge({})
+      end
+
+      it_behaves_like 'tripleo::profile::base::logging::rsyslog::file_input'
+    end
+  end
+end

spec/fixtures/hieradata/step2.yaml

@@ -1,2 +1,12 @@
 ---
 step: 2
+# rsyslog tests
+service_names:
+ - 'horizon'
+tripleo_fluentd_sources_horizon:
+ - path: '/var/log/horizon/test.log'
+   tag: 'openstack.horizon.test'
+ - path: '/var/log/horizon/access.log'
+   tag: 'openstack.horizon.access'
+rsyslog::confdir: /etc/rsyslog.d
+rsyslog::target_file: 50_openstack_logs.conf