os-xenapi: add utility to setup guest VM logs

Add code to copy guest logs to /var/logs/xen/guest and set rotation

Change-Id: Ied7dadad2c3b7b6e9a05cf49d1ad16eac5cf9204

os_xenapi/cmd/bootstrap.py

@@ -24,6 +24,7 @@ import sys
 
 from os_xenapi.utils.common_conf import enable_linux_bridge
 from os_xenapi.utils.common_function import setup_logging
+from os_xenapi.utils.consoles import setup_guest_console_log
 from os_xenapi.utils.himn import config_himn
 from os_xenapi.utils.iptables import config_iptables
 from os_xenapi.utils.sshclient import SSHClient
@@ -107,6 +108,7 @@ def main():
     config_iptables(dom0_client)
     install_plugins_to_dom0(dom0_client)
     enable_linux_bridge(dom0_client)
+    setup_guest_console_log(dom0_client)
 
     # Gather XenAPI relative facts and save them into file.
     get_and_store_facts(dom0_client, facts_file)

os_xenapi/tests/cmd/test_bootstrap.py

@@ -71,6 +71,7 @@ class GetXenapiFactsTestCase(base.TestCase):
 
     @mock.patch.object(bootstrap, '_parse_args')
     @mock.patch.object(bootstrap, 'SSHClient')
+    @mock.patch.object(bootstrap, 'setup_guest_console_log')
     @mock.patch.object(bootstrap, 'config_himn')
     @mock.patch.object(bootstrap, 'config_iptables')
     @mock.patch.object(bootstrap, 'install_plugins_to_dom0')
@@ -78,8 +79,8 @@ class GetXenapiFactsTestCase(base.TestCase):
     @mock.patch.object(bootstrap, 'enable_linux_bridge')
     @mock.patch.object(bootstrap, 'setup_logging')
     def test_bootstrap(self, mock_setup_logging, mock_enable_lbr, mock_facts,
-                       mock_plugin, mock_iptables, mock_himn, mock_client,
-                       mock_parse):
+                       mock_plugin, mock_iptables, mock_himn, mock_guest_log,
+                       mock_client, mock_parse):
         fake_opts = {'himn-ip': '169.254.0.1',
                      'passwd': 'passwd',
                      'user-name': 'root'}
@@ -96,3 +97,4 @@ class GetXenapiFactsTestCase(base.TestCase):
                                       bootstrap.DEF_XENAPI_FACTS_FILE)
         mock_enable_lbr.assert_called_with(mock.sentinel.sshclient)
         mock_setup_logging.assert_called_once_with(log_level=logging.DEBUG)
+        mock_guest_log.assert_called_once_with(mock.sentinel.sshclient)

os_xenapi/utils/consoles.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""console utils
+
+It contains the utilities relative to guest VM console logs collecting."""
+import os
+import sys
+
+from os_xenapi.utils import sshclient
+from os_xenapi.utils import xapi_plugin
+
+OS_XENAPI_PKG = 'os-xenapi'
+
+
+def exit_with_error(err_msg):
+    sys.stderr.write(err_msg)
+    sys.exit(1)
+
+
+def setup_guest_console_log(dom0_client):
+    "Install console logrotate script"
+    dom0_client.ssh('mkdir -p /var/log/xen/guest')
+    dom0_client.ssh('mkdir -p /opt/xensource/bin')
+    is_tmp_dir, os_xenapi_dir = xapi_plugin.get_os_xenapi_dir()
+    dom0_client.scp(
+        os_xenapi_dir + "/os_xenapi/utils/rotate_xen_guest_logs.sh",
+        os.path.join("/opt/xensource/bin/", 'rotate_xen_guest_logs.sh'))
+    dom0_client.ssh('''crontab - << CRONTAB
+* * * * * /opt/xensource/bin/rotate_xen_guest_logs.sh >/dev/null 2>&1
+CRONTAB''')
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 4:
+        exit_with_error("Wrong parameters input.")
+    dom0_himn_ip, user_name, password = sys.argv[1:]
+    try:
+        client = sshclient.SSHClient(dom0_himn_ip, user_name, password)
+    except Exception:
+        exit_with_error("Create connection failed, ip: %(dom0_himn_ip)s,"
+                        " user_name: %(user_name)s" %
+                        {'dom0_himn_ip': dom0_himn_ip, 'user_name': user_name})
+    setup_guest_console_log(client)

os_xenapi/utils/rotate_xen_guest_logs.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+set -eux
+
+# Script to rotate console logs
+#
+# Should be run on Dom0, with cron, every minute:
+# * * * * * /root/rotate_xen_guest_logs.sh
+#
+# Should clear out the guest logs on every boot
+# because the domain ids may get re-used for a
+# different tenant after the reboot
+#
+# /var/log/xen/guest could be mounted into a
+# small loopback device to stop any guest being
+# able to fill dom0 file system
+
+log_dir="/var/log/xen/guest"
+kb=1024
+mb=1048576
+max_size_bytes=$((1*$mb))
+truncated_size_bytes=$((5*$kb))
+syslog_tag='rotate_xen_guest_logs'
+
+log_file_base="${log_dir}/console."
+
+# Only delete log files older than this number of minutes
+# to avoid a race where Xen creates the domain and starts
+# logging before the XAPI VM start returns (and allows us
+# to preserve the log file using last_dom_id)
+min_logfile_age=10
+
+# Ensure logging is setup correctly for all domains
+xenstore-write /local/logconsole/@ "${log_file_base}%d"
+
+# Grab the list of logs now to prevent a race where the domain is
+# started after we get the valid last_dom_ids, but before the logs are
+# deleted.  Add spaces to ensure we can do containment tests below
+current_logs=$(find "$log_dir" -type f)
+
+# Ensure the last_dom_id is set + updated for all running VMs
+for vm in $(xe vm-list power-state=running --minimal | tr ',' ' '); do
+    xe vm-param-set uuid=$vm other-config:last_dom_id=$(xe vm-param-get uuid=$vm param-name=dom-id)
+done
+
+# Get the last_dom_id for all VMs
+valid_last_dom_ids=$(xe vm-list params=other-config --minimal | tr ';,' '\n\n' | grep last_dom_id | sed -e 's/last_dom_id: //g' | xargs)
+echo "Valid dom IDs: $valid_last_dom_ids" | /usr/bin/logger -t $syslog_tag
+
+# Remove old console files that do not correspond to valid last_dom_id's
+allowed_consoles=".*console.\(${valid_last_dom_ids// /\\|}\)$"
+delete_logs=`find "$log_dir" -type f -mmin +${min_logfile_age} -not -regex "$allowed_consoles"`
+for log in $delete_logs; do
+    if echo "$current_logs" | grep -q -w "$log"; then
+        echo "Deleting: $log" | /usr/bin/logger -t $syslog_tag
+        rm $log
+    fi
+done
+
+# Truncate all remaining logs
+for log in `find "$log_dir" -type f -regex '.*console.*' -size +${max_size_bytes}c`; do
+    echo "Truncating log: $log" | /usr/bin/logger -t $syslog_tag
+    tmp="$log.tmp"
+    tail -c $truncated_size_bytes "$log" > "$tmp"
+    mv -f "$tmp" "$log"
+
+    # Notify xen that it needs to reload the file
+    domid="${log##*.}"
+    xenstore-write /local/logconsole/$domid "$log"
+    xenstore-rm /local/logconsole/$domid
+done