Log stdout of HA containers

When podman dropped the journald log-driver we rushed to move to the supported k8s-file driver. This had the side effect of us losing the stdout logs of the HA containers. In fact previously we were easily able to troubleshoot haproxy startup failures just by looking in the journal. These days instead if haproxy fails to start we have no traces whatsoever in the logs, because when a container fails it gets stopped by pacemaker (and consequently removed) and no logs on the system are available any longer. Tested as follows: 1) Redeploy a previously deployed overcloud that did not have the patch and observe that we now log the startup of HA bundles in /var/log/containers/stdouts/*bundle.log [root@controller-0 stdouts]# ls -l *bundle.log |grep -v -e init -e restart -rw-------. 1 root root 16032 Apr 14 14:13 openstack-cinder-volume.log -rw-------. 1 root root 19515 Apr 14 14:00 haproxy-bundle.log -rw-------. 1 root root 10509 Apr 14 14:03 ovn-dbs-bundle.log -rw-------. 1 root root 6451 Apr 14 14:00 redis-bundle.log 2) Deploy a composable HA overcloud from scratch with the patch above and observe that we obtain the stdout on disk. Note that most HA containers log to their usual on-host files just fine, we are mainly missing haproxy logs and/or the kolla startup only of the HA containers. Closes-Bug: #1872734 NB: Cherry-picks had some context change in manifests/profile/pacemaker/cinder/volume_bundle.pp manifests/profile/pacemaker/rabbitmq_bundle.pp manifests/profile/pacemaker/manila/share_bundle.pp Change-Id: I4270b398366e90206adffe32f812632b50df615b (cherry picked from commit 06c4aa7446073022b86c1f034a0c5406f2675ddb)
2020-04-14 11:14:22 +02:00 · 2020-04-14 11:14:22 +02:00 · 7e4aca45fa
commit 7e4aca45fa
parent 1d09aa68fa
11 changed files with 106 additions and 14 deletions
--- a/manifests/profile/pacemaker/cinder/backup_bundle.pp
+++ b/manifests/profile/pacemaker/cinder/backup_bundle.pp
@ -52,6 +52,11 @@
 #   when container_cli is set to podman and 'journald' when it is set to docker.
 #   Defaults to undef
 #
+# [*log_file*]
+#   (optional) Container log file to use. Only relevant when log_driver is
+#   set to 'k8s-file'.
+#   Defaults to '/var/log/containers/stdouts/openstack-cinder-backup.log'
+#
 # [*tls_priorities*]
 #   (optional) Sets PCMK_tls_priorities in /etc/sysconfig/pacemaker when set
 #   Defaults to hiera('tripleo::pacemaker::tls_priorities', undef)
@ -67,6 +72,7 @@ class tripleo::profile::pacemaker::cinder::backup_bundle (
  $docker_environment         = {'KOLLA_CONFIG_STRATEGY' => 'COPY_ALWAYS'},
  $container_backend          = 'docker',
  $log_driver                 = undef,
+  $log_file                   = '/var/log/containers/stdouts/openstack-cinder-backup.log',
  $tls_priorities             = hiera('tripleo::pacemaker::tls_priorities', undef),
  $bundle_user                = 'root',
  $pcs_tries                  = hiera('pcs_tries', 20),
@ -86,6 +92,11 @@ class tripleo::profile::pacemaker::cinder::backup_bundle (
  } else {
    $log_driver_real = $log_driver
  }
+  if $log_driver_real == 'k8s-file' {
+    $log_file_real = " --log-opt path=${log_file}"
+  } else {
+    $log_file_real = ''
+  }

  include ::tripleo::profile::base::cinder::backup

@ -225,7 +236,7 @@ class tripleo::profile::pacemaker::cinder::backup_bundle (
        },
        container_options => 'network=host',
        # lint:ignore:140chars
-        options           => "--ipc=host --privileged=true --user=${bundle_user} --log-driver=${log_driver_real} ${docker_env}${tls_priorities_real}",
+        options           => "--ipc=host --privileged=true --user=${bundle_user} --log-driver=${log_driver_real}${log_file_real} ${docker_env}${tls_priorities_real}",
        # lint:endignore
        run_command       => '/bin/bash /usr/local/bin/kolla_start',
        storage_maps      => $storage_maps,
--- a/manifests/profile/pacemaker/cinder/volume_bundle.pp
+++ b/manifests/profile/pacemaker/cinder/volume_bundle.pp
@ -52,6 +52,11 @@
 #   when container_cli is set to podman and 'journald' when it is set to docker.
 #   Defaults to undef
 #
+# [*log_file*]
+#   (optional) Container log file to use. Only relevant when log_driver is
+#   set to 'k8s-file'.
+#   Defaults to '/var/log/containers/stdouts/openstack-cinder-volume.log'
+#
 # [*tls_priorities*]
 #   (optional) Sets PCMK_tls_priorities in /etc/sysconfig/pacemaker when set
 #   Defaults to hiera('tripleo::pacemaker::tls_priorities', undef)
@ -69,6 +74,7 @@ class tripleo::profile::pacemaker::cinder::volume_bundle (
  $step                       = Integer(hiera('step')),
  $container_backend          = 'docker',
  $log_driver                 = undef,
+  $log_file                   = '/var/log/containers/stdouts/openstack-cinder-volume.log',
  $tls_priorities             = hiera('tripleo::pacemaker::tls_priorities', undef),
  $bundle_user                = 'root',
 ) {
@ -87,7 +93,12 @@ class tripleo::profile::pacemaker::cinder::volume_bundle (
  } else {
    $log_driver_real = $log_driver
  }
-  include ::tripleo::profile::base::cinder::volume
+  if $log_driver_real == 'k8s-file' {
+    $log_file_real = " --log-opt path=${log_file}"
+  } else {
+    $log_file_real = ''
+  }
+  include tripleo::profile::base::cinder::volume

  if $step >= 2 and $pacemaker_master {
    $cinder_volume_short_node_names = hiera('cinder_volume_short_node_names')
@ -226,7 +237,7 @@ class tripleo::profile::pacemaker::cinder::volume_bundle (
        },
        container_options => 'network=host',
        # lint:ignore:140chars
-        options           => "--ipc=host --privileged=true --user=${bundle_user} --log-driver=${log_driver_real} ${docker_env}${tls_priorities_real}",
+        options           => "--ipc=host --privileged=true --user=${bundle_user} --log-driver=${log_driver_real}${log_file_real} ${docker_env}${tls_priorities_real}",
        # lint:endignore
        run_command       => '/bin/bash /usr/local/bin/kolla_start',
        storage_maps      => $storage_maps,
--- a/manifests/profile/pacemaker/database/mysql_bundle.pp
+++ b/manifests/profile/pacemaker/database/mysql_bundle.pp
@ -128,6 +128,11 @@
 #   when container_cli is set to podman and 'journald' when it is set to docker.
 #   Defaults to undef
 #
+# [*log_file*]
+#   (optional) Container log file to use. Only relevant when log_driver is
+#   set to 'k8s-file'.
+#   Defaults to '/var/log/containers/stdouts/galera-bundle.log'
+#
 # [*tls_priorities*]
 #   (optional) Sets PCMK_tls_priorities in /etc/sysconfig/pacemaker when set
 #   Defaults to hiera('tripleo::pacemaker::tls_priorities', undef)
@ -157,6 +162,7 @@ class tripleo::profile::pacemaker::database::mysql_bundle (
  $mysql_auth_ed25519             = hiera('mysql_auth_ed25519', false),
  $container_backend              = 'docker',
  $log_driver                     = undef,
+  $log_file                       = '/var/log/containers/stdouts/galera-bundle.log',
  $tls_priorities                 = hiera('tripleo::pacemaker::tls_priorities', undef),
  $bundle_user                    = 'root',
  $pcs_tries                      = hiera('pcs_tries', 20),
@ -177,6 +183,11 @@ class tripleo::profile::pacemaker::database::mysql_bundle (
  } else {
    $log_driver_real = $log_driver
  }
+  if $log_driver_real == 'k8s-file' {
+    $log_file_real = " --log-opt path=${log_file}"
+  } else {
+    $log_file_real = ''
+  }
  # FQDN are lowercase in /etc/hosts, so are pacemaker node names
  $galera_node_names_lookup = downcase(hiera('mysql_short_node_names_override',
                                              hiera('mysql_short_node_names', $::hostname)))
@ -450,7 +461,7 @@ MYSQL_HOST=localhost\n",
        },
        container_options => 'network=host',
        # lint:ignore:140chars
-        options           => "--user=${bundle_user} --log-driver=${log_driver_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS${tls_priorities_real}",
+        options           => "--user=${bundle_user} --log-driver=${log_driver_real}${log_file_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS${tls_priorities_real}",
        # lint:endignore
        run_command       => '/bin/bash /usr/local/bin/kolla_start',
        network           => "control-port=${control_port}",
--- a/manifests/profile/pacemaker/database/redis_bundle.pp
+++ b/manifests/profile/pacemaker/database/redis_bundle.pp
@ -100,6 +100,11 @@
 #   when container_cli is set to podman and 'journald' when it is set to docker.
 #   Defaults to undef
 #
+# [*log_file*]
+#   (optional) Container log file to use. Only relevant when log_driver is
+#   set to 'k8s-file'.
+#   Defaults to '/var/log/containers/stdouts/redis-bundle.log'
+#
 # [*tls_priorities*]
 #   (optional) Sets PCMK_tls_priorities in /etc/sysconfig/pacemaker when set
 #   Defaults to hiera('tripleo::pacemaker::tls_priorities', undef)
@ -117,6 +122,7 @@ class tripleo::profile::pacemaker::database::redis_bundle (
  $container_backend         = 'docker',
  $pcs_tries                 = hiera('pcs_tries', 20),
  $log_driver                = undef,
+  $log_file                  = '/var/log/containers/stdouts/redis-bundle.log',
  $step                      = Integer(hiera('step')),
  $redis_network             = hiera('redis_network', undef),
  $extra_config_file         = '/etc/redis-tls.conf',
@ -143,6 +149,11 @@ class tripleo::profile::pacemaker::database::redis_bundle (
  } else {
    $log_driver_real = $log_driver
  }
+  if $log_driver_real == 'k8s-file' {
+    $log_file_real = " --log-opt path=${log_file}"
+  } else {
+    $log_file_real = ''
+  }
  if $enable_internal_tls {
    if !$redis_network {
      fail('redis_network is not set in the hieradata.')
@ -354,7 +365,7 @@ slave-announce-port ${local_tuple[0][2]}
        },
        container_options => 'network=host',
        # lint:ignore:140chars
-        options           => "--user=${bundle_user} --log-driver=${log_driver_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS${tls_priorities_real}",
+        options           => "--user=${bundle_user} --log-driver=${log_driver_real}${log_file_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS${tls_priorities_real}",
        # lint:endignore
        run_command       => '/bin/bash /usr/local/bin/kolla_start',
        network           => "control-port=${redis_docker_control_port}",
--- a/manifests/profile/pacemaker/haproxy_bundle.pp
+++ b/manifests/profile/pacemaker/haproxy_bundle.pp
@ -76,6 +76,11 @@
 #   when container_cli is set to podman and 'journald' when it is set to docker.
 #   Defaults to undef
 #
+# [*log_file*]
+#   (optional) Container log file to use. Only relevant when log_driver is
+#   set to 'k8s-file'.
+#   Defaults to '/var/log/containers/stdouts/haproxy-bundle.log'
+#
 # [*tls_priorities*]
 #   (optional) Sets PCMK_tls_priorities in /etc/sysconfig/pacemaker when set
 #   Defaults to hiera('tripleo::pacemaker::tls_priorities', undef)
@ -109,6 +114,7 @@ class tripleo::profile::pacemaker::haproxy_bundle (
  $tls_priorities           = hiera('tripleo::pacemaker::tls_priorities', undef),
  $bundle_user              = 'root',
  $log_driver               = undef,
+  $log_file                 = '/var/log/containers/stdouts/haproxy-bundle.log',
  $step                     = Integer(hiera('step')),
  $pcs_tries                = hiera('pcs_tries', 20),
 ) {
@ -129,6 +135,12 @@ class tripleo::profile::pacemaker::haproxy_bundle (
  } else {
    $log_driver_real = $log_driver
  }
+  if $log_driver_real == 'k8s-file' {
+    $log_file_real = " --log-opt path=${log_file}"
+  } else {
+    $log_file_real = ''
+  }
+
  if $step >= 2 and $enable_load_balancer {
    if $pacemaker_master {
      if (hiera('haproxy_short_node_names_override', undef)) {
@ -276,7 +288,7 @@ class tripleo::profile::pacemaker::haproxy_bundle (
        location_rule     => $haproxy_location_rule,
        container_options => 'network=host',
        # lint:ignore:140chars
-        options           => "--user=${bundle_user} --log-driver=${log_driver_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS${tls_priorities_real}",
+        options           => "--user=${bundle_user} --log-driver=${log_driver_real}${log_file_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS${tls_priorities_real}",
        # lint:endignore
        run_command       => '/bin/bash /usr/local/bin/kolla_start',
        storage_maps      => merge($storage_maps, $cert_storage_maps, $storage_maps_internal_tls),
--- a/manifests/profile/pacemaker/manila/share_bundle.pp
+++ b/manifests/profile/pacemaker/manila/share_bundle.pp
@ -56,6 +56,11 @@
 #   when container_cli is set to podman and 'journald' when it is set to docker.
 #   Defaults to undef
 #
+# [*log_file*]
+#   (optional) Container log file to use. Only relevant when log_driver is
+#   set to 'k8s-file'.
+#   Defaults to '/var/log/containers/stdouts/openstack-manila-share.log'
+#
 # [*tls_priorities*]
 #   (optional) Sets PCMK_tls_priorities in /etc/sysconfig/pacemaker when set
 #   Defaults to hiera('tripleo::pacemaker::tls_priorities', undef)
@ -74,6 +79,7 @@ class tripleo::profile::pacemaker::manila::share_bundle (
  $tls_priorities             = hiera('tripleo::pacemaker::tls_priorities', undef),
  $bundle_user                = 'root',
  $log_driver                 = undef,
+  $log_file                   = '/var/log/containers/stdouts/openstack-manila-share.log',
  $pcs_tries                  = hiera('pcs_tries', 20),
  $step                       = Integer(hiera('step')),
 ) {
@ -92,7 +98,12 @@ class tripleo::profile::pacemaker::manila::share_bundle (
  } else {
    $log_driver_real = $log_driver
  }
-  include ::tripleo::profile::base::manila::share
+  if $log_driver_real == 'k8s-file' {
+    $log_file_real = " --log-opt path=${log_file}"
+  } else {
+    $log_file_real = ''
+  }
+  include tripleo::profile::base::manila::share

  if $step >= 2 and $pacemaker_master {
    $manila_share_short_node_names = hiera('manila_share_short_node_names')
@ -249,7 +260,7 @@ class tripleo::profile::pacemaker::manila::share_bundle (
        },
        container_options => 'network=host',
        # lint:ignore:140chars
-        options           => "--ipc=host --privileged=true --user=${bundle_user} --log-driver=${log_driver_real} ${docker_env}${tls_priorities_real}",
+        options           => "--ipc=host --privileged=true --user=${bundle_user} --log-driver=${log_driver_real}${log_file_real} ${docker_env}${tls_priorities_real}",
        # lint:endignore
        run_command       => '/bin/bash /usr/local/bin/kolla_start',
        storage_maps      => $storage_maps,
--- a/manifests/profile/pacemaker/ovn_dbs_bundle.pp
+++ b/manifests/profile/pacemaker/ovn_dbs_bundle.pp
@ -69,6 +69,11 @@
 #   when container_cli is set to podman and 'journald' when it is set to docker.
 #   Defaults to undef
 #
+# [*log_file*]
+#   (optional) Container log file to use. Only relevant when log_driver is
+#   set to 'k8s-file'.
+#   Defaults to '/var/log/containers/stdouts/ovn-dbs-bundle.log'
+#
 # [*tls_priorities*]
 #   (optional) Sets PCMK_tls_priorities in /etc/sysconfig/pacemaker when set
 #   Defaults to hiera('tripleo::pacemaker::tls_priorities', undef)
@ -112,6 +117,7 @@ class tripleo::profile::pacemaker::ovn_dbs_bundle (
  $tls_priorities           = hiera('tripleo::pacemaker::tls_priorities', undef),
  $bundle_user              = undef,
  $log_driver               = undef,
+  $log_file                 = '/var/log/containers/stdouts/ovn-dbs-bundle.log',
  $enable_internal_tls      = hiera('enable_internal_tls', false),
  $ca_file                  = undef,
  $dbs_timeout              = hiera('tripleo::profile::pacemaker::ovn_dbs_bundle::dbs_timeout', 60),
@ -133,6 +139,11 @@ class tripleo::profile::pacemaker::ovn_dbs_bundle (
  } else {
    $log_driver_real = $log_driver
  }
+  if $log_driver_real == 'k8s-file' {
+    $log_file_real = " --log-opt path=${log_file}"
+  } else {
+    $log_file_real = ''
+  }
  if $step >= 3 {

    if $pacemaker_master {
@ -235,7 +246,7 @@ nb_master_protocol=ssl sb_master_protocol=ssl"
        location_rule     => $ovn_dbs_location_rule,
        container_options => 'network=host',
        # lint:ignore:140chars
-        options           => "${bundle_user_real}--log-driver=${log_driver_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS${tls_priorities_real}",
+        options           => "${bundle_user_real}--log-driver=${log_driver_real}${log_file_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS${tls_priorities_real}",
        # lint:endignore
        run_command       => '/bin/bash /usr/local/bin/kolla_start',
        network           => "control-port=${ovn_dbs_control_port}",
--- a/manifests/profile/pacemaker/rabbitmq_bundle.pp
+++ b/manifests/profile/pacemaker/rabbitmq_bundle.pp
@ -88,6 +88,11 @@
 #   when container_cli is set to podman and 'journald' when it is set to docker.
 #   Defaults to undef
 #
+# [*log_file*]
+#   (optional) Container log file to use. Only relevant when log_driver is
+#   set to 'k8s-file'.
+#   Defaults to '/var/log/containers/stdouts/rabbitmq-bundle.log'
+#
 # [*tls_priorities*]
 #   (optional) Sets PCMK_tls_priorities in /etc/sysconfig/pacemaker when set
 #   Defaults to hiera('tripleo::pacemaker::tls_priorities', undef)
@ -115,6 +120,7 @@ class tripleo::profile::pacemaker::rabbitmq_bundle (
  $step                         = Integer(hiera('step')),
  $container_backend            = 'docker',
  $log_driver                   = undef,
+  $log_file                     = '/var/log/containers/stdouts/rabbitmq-bundle.log',
  $tls_priorities               = hiera('tripleo::pacemaker::tls_priorities', undef),
  $bundle_user                  = 'root',
 ) {
@ -151,7 +157,12 @@ class tripleo::profile::pacemaker::rabbitmq_bundle (
  } else {
    $log_driver_real = $log_driver
  }
-  include ::tripleo::profile::base::rabbitmq
+  if $log_driver_real == 'k8s-file' {
+    $log_file_real = " --log-opt path=${log_file}"
+  } else {
+    $log_file_real = ''
+  }
+  include tripleo::profile::base::rabbitmq

  file { '/var/lib/rabbitmq/.erlang.cookie':
    ensure  => file,
@ -289,7 +300,7 @@ class tripleo::profile::pacemaker::rabbitmq_bundle (
        },
        container_options => 'network=host',
        # lint:ignore:140chars
-        options           => "--user=${bundle_user} --log-driver=${log_driver_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS -e LANG=en_US.UTF-8 -e LC_ALL=en_US.UTF-8${tls_priorities_real}",
+        options           => "--user=${bundle_user} --log-driver=${log_driver_real}${log_file_real} -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS -e LANG=en_US.UTF-8 -e LC_ALL=en_US.UTF-8${tls_priorities_real}",
        # lint:endignore
        run_command       => '/bin/bash /usr/local/bin/kolla_start',
        network           => "control-port=${rabbitmq_docker_control_port}",
--- a/spec/classes/tripleo_profile_pacemaker_cinder_backup_bundle_spec.rb
+++ b/spec/classes/tripleo_profile_pacemaker_cinder_backup_bundle_spec.rb
@ -82,12 +82,13 @@ describe 'tripleo::profile::pacemaker::cinder::backup_bundle' do
            :docker_volumes     => ['/src/1:/tgt/1', '/src/2:/tgt/2:ro', '/src/3:/tgt/3:ro,z'],
            :docker_environment => ['RIGHT=LEFT', 'UP=DOWN'],
            :log_driver         => 'k8s-file',
+            :log_file           => '/var/log/containers/stdouts/cinder_backup.log'
          })
        end
        it 'should create custom cinder-backup resource bundle' do
          is_expected.to contain_pacemaker__resource__bundle('openstack-cinder-backup').with(
            :image        => 'c-bak-docker-image',
-            :options      => '--ipc=host --privileged=true --user=root --log-driver=k8s-file -e RIGHT=LEFT -e UP=DOWN',
+            :options      => '--ipc=host --privileged=true --user=root --log-driver=k8s-file --log-opt path=/var/log/containers/stdouts/cinder_backup.log -e RIGHT=LEFT -e UP=DOWN',
            :storage_maps => {
              'cinder-backup-src-1' => {
                'source-dir' => '/src/1',
--- a/spec/classes/tripleo_profile_pacemaker_cinder_volume_bundle_spec.rb
+++ b/spec/classes/tripleo_profile_pacemaker_cinder_volume_bundle_spec.rb
@ -88,12 +88,13 @@ describe 'tripleo::profile::pacemaker::cinder::volume_bundle' do
            :docker_volumes     => ['/src/1:/tgt/1', '/src/2:/tgt/2:ro', '/src/3:/tgt/3:ro,z'],
            :docker_environment => ['RIGHT=LEFT', 'UP=DOWN'],
            :log_driver         => 'k8s-file',
+            :log_file           => '/var/log/containers/stdouts/cinder_volume.log'
          })
        end
        it 'should create custom cinder-volume resource bundle' do
          is_expected.to contain_pacemaker__resource__bundle('openstack-cinder-volume').with(
            :image        => 'c-vol-docker-image',
-            :options      => '--ipc=host --privileged=true --user=root --log-driver=k8s-file -e RIGHT=LEFT -e UP=DOWN',
+            :options      => '--ipc=host --privileged=true --user=root --log-driver=k8s-file --log-opt path=/var/log/containers/stdouts/cinder_volume.log -e RIGHT=LEFT -e UP=DOWN',
            :storage_maps => {
              'cinder-volume-src-1' => {
                'source-dir' => '/src/1',
--- a/spec/classes/tripleo_profile_pacemaker_manila_share_bundle_spec.rb
+++ b/spec/classes/tripleo_profile_pacemaker_manila_share_bundle_spec.rb
@ -101,12 +101,13 @@ describe 'tripleo::profile::pacemaker::manila::share_bundle' do
            :docker_volumes     => ['/src/1:/tgt/1', '/src/2:/tgt/2:ro', '/src/3:/tgt/3:ro,z'],
            :docker_environment => ['RIGHT=LEFT', 'UP=DOWN'],
            :log_driver         => 'k8s-file',
+            :log_file           => '/var/log/containers/stdouts/manila_share.log'
          })
        end
        it 'should create custom manila-share resource bundle' do
          is_expected.to contain_pacemaker__resource__bundle('openstack-manila-share').with(
            :image        => 'manila-share-image',
-            :options      => '--ipc=host --privileged=true --user=root --log-driver=k8s-file -e RIGHT=LEFT -e UP=DOWN',
+            :options      => '--ipc=host --privileged=true --user=root --log-driver=k8s-file --log-opt path=/var/log/containers/stdouts/manila_share.log -e RIGHT=LEFT -e UP=DOWN',
            :storage_maps => {
              'manila-share-src-1' => {
                'source-dir' => '/src/1',