Merge "Base profile for rsyslogd"

Puppetfile_extras

@@ -54,6 +54,10 @@ mod 'opendaylight',
   :git => 'https://git.opendaylight.org/gerrit/integration/packaging/puppet-opendaylight',
   :ref => 'master'
 
+mod 'rsyslog',
+  :git => 'https://github.com/voxpupuli/puppet-rsyslog',
+  :ref => 'master'
+
 mod 'ssh',
   :git => 'https://github.com/saz/puppet-ssh',
   :ref => 'v3.0.1'

manifests/profile/base/logging/rsyslog.pp

@@ -0,0 +1,54 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: tripleo::profile::base::logging::rsyslog
+#
+# rsyslogd configuration for TripleO
+#
+# === Parameters
+#
+# [*step*]
+#   (Optional) String. The current step of the deployment
+#   Defaults to hiera('step')
+#
+# [*elasticsearch*]
+#   (Optional) Hash. Configuration for output plugin omelasticsearch.
+#
+# [*service_names*]
+#   (Optional) List of services enabled on the current role. This is used
+#   to obtain per-service configuration information.
+#
+class tripleo::profile::base::logging::rsyslog (
+  $step            = Integer(hiera('step')),
+  $elasticsearch   = undef,
+  $service_names   = hiera('service_names', [])
+) {
+  if $step >= 2 {
+    $modules = {
+      'imfile'          => {},
+      'omelasticsearch' => {},
+    }
+    $actions = {
+      'elasticsearch' => {
+        'type'   => 'omelasticsearch',
+        'config' => $elasticsearch,
+      }
+    }
+    class { '::rsyslog::server':
+      modules => $modules,
+      actions => $actions
+    }
+    tripleo::profile::base::logging::rsyslog::file_input{$service_names: }
+  }
+}

manifests/profile/base/logging/rsyslog/file_input.pp

@@ -0,0 +1,52 @@
+# This is used to look up a list of service-specific rsyslogd configurations
+# in the hiera data provided by THT.
+#
+# [*sources*]
+#   (Optional) List of hashes. Contains configuration of file inputs for given service.
+#   Defaults to hiera("tripleo_logging_sources_${title}", undef)
+#
+# [*default_startmsg*]
+#   (Optional) String. Default POSIX ERE for start of log record. The default enables to forward
+#   multiline tracebacks for most of OpenStack services. It can be overridden either by this
+#   parameter for all file inputs or in each file input separately in THT parameters
+#   tripleo_logging_sources_<service name>.
+#   Defaults to '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) '
+#
+define tripleo::profile::base::logging::rsyslog::file_input (
+  $sources = hiera("tripleo_logging_sources_${title}", undef),
+  $default_startmsg = '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) '
+) {
+  # TODO(mmagr): As a deprecation path for fluentd we gonna use the fluentd THT parameters first.
+  # We can remove below few lines after fluentd is removed and it's template outputs
+  # are renamed to provider-agnostic name in all composable services
+  if !$sources {
+    $fluentd_sources = hiera("tripleo_fluentd_sources_${title}", [])
+    $rsyslog_sources = $fluentd_sources.reduce([]) |$memo, $config| {
+      if $config['startmsg.regex'] {
+        $startmsg = $config['startmsg.regex']
+      } else {
+        $startmsg = $default_startmsg
+      }
+      $memo + [{'file' => $config['path'], 'tag' => $config['tag'], 'startmsg.regex' => $startmsg }]
+    }
+  } else {
+    $rsyslog_sources = $sources.reduce([]) |$memo, $config| {
+      if ! $config['startmsg.regex'] {
+        $record = $config + {'startmsg.regex' => $default_startmsg}
+      } else {
+        $record = $config
+      }
+      $memo + [$record]
+    }
+  }
+
+  $rsyslog_sources.each |$config| {
+    rsyslog::component::input{ "${title}_${config['tag']}":
+      priority => $::rsyslog::input_priority,
+      target   => $::rsyslog::target_file,
+      confdir  => $::rsyslog::confdir,
+      type     => 'imfile',
+      config   => $config
+    }
+  }
+}

releasenotes/notes/rsyslog-205c11903ed92bdf.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Adds profile for rsyslogd composable service which aims to replace fluentd
+    with the same behaviour. This means that rsyslog will be tailing OpenStack
+    log files and forwarding it to central log collector (ELK)

spec/classes/tripleo_profile_base_logging_rsyslog_spec.rb

@@ -0,0 +1,84 @@
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+require 'spec_helper'
+
+horizon_access_log_conf = <<-EOS
+# horizon_openstack.horizon.access
+input(type="imfile"
+  file="/var/log/horizon/access.log"
+  tag="openstack.horizon.access"
+  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
+)
+EOS
+horizon_test_log_conf = <<-EOS
+# horizon_openstack.horizon.test
+input(type="imfile"
+  file="/var/log/horizon/test.log"
+  tag="openstack.horizon.test"
+  startmsg.regex="^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]+ [0-9]+)? (DEBUG|INFO|WARNING|ERROR) "
+)
+EOS
+elastic_conf = <<-EOS
+# elasticsearch
+action(type="omelasticsearch"
+    name="elasticsearch"
+)
+EOS
+
+describe 'tripleo::profile::base::logging::rsyslog' do
+  shared_examples_for 'tripleo::profile::base::logging::rsyslog' do
+    before :each do
+      facts.merge!({ :step => params[:step] })
+    end
+
+    context 'on step 2' do
+      let(:params) { { :step => 2 } }
+
+      it 'should generate a rsyslog config file for horizon from hieradata' do
+        should contain_concat__fragment("rsyslog::component::module::imfile").with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => "module(load=\"imfile\")\n",
+        })
+        should contain_concat__fragment("rsyslog::component::module::omelasticsearch").with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => "module(load=\"omelasticsearch\")\n",
+        })
+        should contain_concat__fragment('rsyslog::component::input::horizon_openstack.horizon.access').with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => horizon_access_log_conf,
+        })
+        should contain_concat__fragment('rsyslog::component::input::horizon_openstack.horizon.test').with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => horizon_test_log_conf,
+        })
+        should contain_concat__fragment("rsyslog::component::action::elasticsearch").with({
+          :target => '/etc/rsyslog.d/50_openstack_logs.conf',
+          :content => elastic_conf,
+        })
+      end
+    end
+  end
+
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let (:facts) {
+        facts
+      }
+      it_behaves_like 'tripleo::profile::base::logging::rsyslog'
+    end
+  end
+end

spec/defines/tripleo_profile_base_logging_rsyslog_file_input_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+foo_log_conf = <<-EOS
+# foobar_foo
+input(type="imfile"
+  file="/path/to/foo.log"
+  tag="foo"
+  startmsg.regex="test"
+)
+EOS
+bar_log_conf = <<-EOS
+# foobar_bar
+input(type="imfile"
+  file="/path/to/bar.log"
+  tag="bar"
+  startmsg.regex="baz"
+)
+EOS
+
+describe 'tripleo::profile::base::logging::rsyslog::file_input' do
+  let(:title) { 'foobar' }
+
+  let :pre_condition do
+    'include ::rsyslog::server'
+  end
+
+  let :params do {
+    'sources' => [
+      {'file' => '/path/to/foo.log', 'tag' => 'foo'},
+      {'file' => '/path/to/bar.log', 'tag' => 'bar', 'startmsg.regex' => 'baz'}
+    ],
+    'default_startmsg' => 'test'
+  }
+  end
+
+  shared_examples_for 'tripleo::profile::base::logging::rsyslog::file_input' do
+    context 'with basic parameters to configure rsyslog file inputs' do
+      it 'should configure the given file inputs' do
+        should contain_concat__fragment('rsyslog::component::input::foobar_foo').with({
+          :target => '/etc/rsyslog.d/50_rsyslog.conf',
+          :content => foo_log_conf,
+        })
+        should contain_concat__fragment('rsyslog::component::input::foobar_bar').with({
+          :target => '/etc/rsyslog.d/50_rsyslog.conf',
+          :content => bar_log_conf,
+        })
+      end
+    end
+  end
+
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let(:facts) do
+        facts.merge({})
+      end
+
+      it_behaves_like 'tripleo::profile::base::logging::rsyslog::file_input'
+    end
+  end
+end

spec/fixtures/hieradata/step2.yaml

@@ -1,2 +1,12 @@
 ---
 step: 2
+# rsyslog tests
+service_names:
+ - 'horizon'
+tripleo_fluentd_sources_horizon:
+ - path: '/var/log/horizon/test.log'
+   tag: 'openstack.horizon.test'
+ - path: '/var/log/horizon/access.log'
+   tag: 'openstack.horizon.access'
+rsyslog::confdir: /etc/rsyslog.d
+rsyslog::target_file: 50_openstack_logs.conf