Introduce nova virtlogd wrapper

When nova_virtlogd container gets restarted the instance console auth files
will not be reopened again by virtlogd. As a result either instances need
to be restarted or live migrated to a different compute node to get new
console logs messages logged again.
Usually on receipt of SIGUSR1, virtlogd will re-exec() its binary, while
maintaining all current logs and clients. This allows for live upgrades of
the virtlogd service on non containerized environments where updates just
by doing an RPM update.
To reduce the likelihood in a containerized environment virtlogd should
only be restarted on manual request, or on compute node reboot. It should
not be restarted on a minor update without migration off instances.
This introduces a nova_virtlogd_wrapper container and virtlogd wrapper
script, to only restart virtlogd on either manual or compute node restart.

Closes-Bug: #1838272

Change-Id: I5192f8c306422d0966eef7544f719cc21172bd92
Co-Authored-By: Rajesh Tailor <ratailor@redhat.com>

manifests/profile/base/nova/virtlogd_wrapper.pp

@@ -0,0 +1,55 @@
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: tripleo::profile::base::nova::virtlogd_wrapper
+#
+# Generates wrapper scripts for running virtlogd in container.
+#
+# === Parameters
+#
+# [*enable_wrapper*]
+#  (Optional) If true, generates a wrapper for running virtlogd in
+#  a docker container.
+#  Defaults to false
+#
+# [*virtlogd_process_wrapper*]
+#   (Optional) Filename for virtlogd wrapper in the specified file.
+#   Defaults to undef
+#
+# [*virtlogd_image*]
+#   (Optional) Docker image name for virtlogd. Required if
+#   virtlogd_wrapper is set.
+#   Defaults to undef
+#
+# [*debug*]
+#   (Optional) Debug messages for the wrapper scripts.
+#   Defaults to False.
+#
+class tripleo::profile::base::nova::virtlogd_wrapper (
+  $enable_wrapper           = false,
+  $virtlogd_process_wrapper = undef,
+  $virtlogd_image           = undef,
+  Boolean $debug            = false,
+) {
+  if $enable_wrapper {
+    unless $virtlogd_image and $virtlogd_process_wrapper{
+      fail('The docker image for virtlogd and wrapper filename must be provided when generating virtlogd wrappers')
+    }
+    tripleo::profile::base::nova::wrappers::virtlogd{'nova_virtlogd_wrapper':
+      virtlogd_process_wrapper => $virtlogd_process_wrapper,
+      virtlogd_image           => $virtlogd_image,
+      debug                    => $debug,
+    }
+  }
+}

manifests/profile/base/nova/wrappers/virtlogd.pp

@@ -0,0 +1,43 @@
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == define: tripleo::profile::base::nova::wrappers::virtlogd
+#
+# Generates wrapper script for running virtlogd in a container.
+#
+# === Parameters
+#
+# [*virtlogd_process_wrapper*]
+#   Filename for virtlogd wrapper script.
+#
+# [*virtlogd_image*]
+#   Docker image name for virtlogd.
+#
+# [*debug*]
+#   Enable debug messages for the wrapper script.
+#
+define tripleo::profile::base::nova::wrappers::virtlogd (
+  $virtlogd_process_wrapper,
+  $virtlogd_image,
+  Boolean $debug,
+) {
+    file { $virtlogd_process_wrapper:
+      ensure  => file,
+      mode    => '0755',
+      content => epp('tripleo/nova/virtlogd.epp', {
+        'image_name' => $virtlogd_image,
+        'debug'      => $debug,
+        })
+    }
+}

releasenotes/notes/nova_virtlogd_wrapper-43c6c319db2a36ef.yaml

@@ -0,0 +1,16 @@
+---
+features:
+  - |
+    When nova_virtlogd container gets restarted the instance console auth files
+    will not be reopened again by virtlogd. As a result either instances need
+    to be restarted or live migrated to a different compute node to get new
+    console logs messages logged again.
+    Usually on receipt of SIGUSR1, virtlogd will re-exec() its binary, while
+    maintaining all current logs and clients. This allows for live upgrades of
+    the virtlogd service on non containerized environments where updates just
+    by doing an RPM update.
+    To reduce the likelihood in a containerized environment virtlogd should
+    only be restarted on manual request, or on compute node reboot. It should
+    not be restarted on a minor update without migration off instances.
+    This introduces a nova_virtlogd_wrapper container and virtlogd wrapper
+    script, to only restart virtlogd on either manual or compute node restart.

templates/nova/virtlogd.epp

@@ -0,0 +1,52 @@
+<%- | String  $image_name    = '',
+      Boolean $debug
+| -%>
+#!/bin/bash
+<%- if $debug { -%>set -x<%- } -%>
+ARGS="$@"
+NAME=nova_virtlogd
+CMD='/usr/sbin/virtlogd --config /etc/libvirt/virtlogd.conf'
+CLI="nsenter --preserve-credentials -m -t 1 podman"
+LOGGING="--log-driver k8s-file --log-opt path=/var/log/containers/stdouts/${NAME}.log"
+VIRTLOGD_CONTAINER=$(${CLI} ps -a --filter name="^${NAME}$" --format '{{.ID}}:{{.Status}}')
+
+CONTAINER_ID=$(echo $VIRTLOGD_CONTAINER | awk -F: '{print $1}')
+CONTAINER_STATUS=$(echo $VIRTLOGD_CONTAINER | awk -F: '{print $2}')
+CONTAINER_STATUS_SHORT=$(echo $CONTAINER_STATUS | awk '{print $1}')
+
+case ${CONTAINER_STATUS_SHORT} in
+    Up | up | UP)
+        echo "Container ${NAME} with id ${CONTAINER_ID} is already running!"
+        ;;
+    Exited| exited| EXITED)
+        echo "Removing orphaned ${NAME} container ${CONTAINER_ID}"
+        $CLI stop ${CONTAINER_ID} || true
+        $CLI rm -f ${CONTAINER_ID} || true
+        ;&
+    *)
+        echo "Starting a new child container ${NAME}"
+        $CLI run --rm --detach ${LOGGING} \
+            -v /etc/hosts:/etc/hosts:ro \
+            -v /etc/localtime:/etc/localtime:ro \
+            -v /var/lib/config-data/nova_libvirt/etc/libvirt:/etc/libvirt:ro \
+            -v /lib/modules:/lib/modules:ro \
+            -v /dev/log:/dev/log \
+            -v /run:/run \
+            -v /sys/fs/cgroup:/sys/fs/cgroup \
+            -v /var/run/libvirt:/var/run/libvirt:shared \
+            -v /var/lib/libvirt:/var/lib/libvirt \
+            -v /var/log/containers/libvirt:/var/log/libvirt \
+            -v /var/log/libvirt/qemu:/var/log/libvirt/qemu \
+            -v /var/lib/nova:/var/lib/nova:shared \
+            --net host \
+            --pid host \
+            --privileged \
+            -u root \
+            --name $NAME \
+            <%=$image_name%> \
+            $CMD $ARGS
+        ;;
+
+esac
+
+sleep infinity