From 892e63ba6ed6d6a3fcdbacb06437f61ea16ce89b Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Thu, 18 Jun 2015 15:49:17 +1000
Subject: [PATCH] Add option to setup netconsole to remote destination

Debugging a kernel oops or other issues where jenkins drops connection
with the host is very difficult.  The console screen requires one of
the friendly infra-* staff to examine, but even then it's small and
can scroll off important information or blank out.  Also, if you're
killing the network connection you're left with no information what
might be going on in the host.

After hitting this issue recently, I did some change-specific stuff to
setup a netconsole on the host and send the logs out that way.  It
ended up working well and gave me a full backtrace of the oops, which
cut down the debugging time considerably.

This wraps the setup into a function so you just need to specify
DEVSTACK_GATE_NETCONSOLE=123.123.123.123:6666 and it will setup the
host to send logs there.  It is intended for short-term use while
you're debugging specific problems with a job.

Change-Id: Iebc8e1812d1441aba7c18d3e7c982b620b5198a0
---
 devstack-vm-gate-wrap.sh | 21 +++++++++++++++++++++
 functions.sh             | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh
index 95a7e455..c7fb4fdb 100755
--- a/devstack-vm-gate-wrap.sh
+++ b/devstack-vm-gate-wrap.sh
@@ -391,6 +391,27 @@ export DEVSTACK_GATE_TOPOLOGY=${DEVSTACK_GATE_TOPOLOGY:-aio}
 # for jobs that know exactly which repos they need.
 export DEVSTACK_GATE_PROJECTS_OVERRIDE=${DEVSTACK_GATE_PROJECTS_OVERRIDE:-""}
 
+# Set this to enable remote logging of the console via UDP packets to
+# a specified ipv4 ip:port (note; not hostname -- ip address only).
+# This can be extremely useful if a host is oopsing or dropping off
+# the network amd you are not getting any useful logs from jenkins.
+#
+# To capture these logs, enable a netcat/socat type listener to
+# capture UDP packets at the specified remote ip.  For example:
+#
+#  $ nc -v -u -l -p 6666 | tee save-output.log
+# or
+#  $ socat udp-recv:6666 - | tee save-output.log
+#
+# One further trick is to send interesting data to /dev/ksmg; this
+# data will get out over the netconsole even if the main interfaces
+# have been disabled, etc.  e.g.
+#
+#  $ ip addr | sudo tee /dev/ksmg
+#
+export DEVSTACK_GATE_NETCONSOLE=${DEVSTACK_GATE_NETCONSOLE:-""}
+enable_netconsole
+
 if [ -n "$DEVSTACK_GATE_PROJECTS_OVERRIDE" ]; then
     PROJECTS=$DEVSTACK_GATE_PROJECTS_OVERRIDE
 fi
diff --git a/functions.sh b/functions.sh
index 9fdaa7a3..b5d1cd8c 100644
--- a/functions.sh
+++ b/functions.sh
@@ -827,6 +827,45 @@ function remote_copy_file {
     scp $ssh_opts "$src" "$dest"
 }
 
+# enable_netconsole
+function enable_netconsole {
+
+    # do nothing if not set
+    if [[ $DEVSTACK_GATE_NETCONSOLE = "" ]]; then
+        return
+    fi
+
+    local remote_ip=$(echo $DEVSTACK_GATE_NETCONSOLE | awk -F: -e '{print $1}')
+    local remote_port=$(echo $DEVSTACK_GATE_NETCONSOLE | awk -F: -e '{print $2}')
+
+    # netconsole requires the device to send and the destitation MAC,
+    # which is obviously on the same subnet.  The way to get packets
+    # out to the world is specify the default gw as the remote
+    # destination.
+    local default_gw=$(ip route | grep default | awk '{print $3}')
+    local gw_mac=$(arp $default_gw | grep $default_gw | awk '{print $3}')
+    local gw_dev=$(ip route | grep default | awk '{print $5}')
+
+    # turn up message output
+    sudo dmesg -n 8
+
+    sudo modprobe configfs
+    sudo modprobe netconsole
+
+    sudo mount none -t configfs /sys/kernel/config
+
+    sudo mkdir /sys/kernel/config/netconsole/target1
+
+    pushd /sys/kernel/config/netconsole/target1
+    echo "$gw_dev" | sudo tee ./dev_name
+    echo "$remote_ip" | sudo tee ./remote_ip
+    echo "$gw_mac" | sudo tee ./remote_mac
+    echo "$remote_port" | sudo tee ./remote_port
+    echo 1 | sudo tee ./enabled
+    popd
+}
+
+
 # This function creates an internal gre bridge to connect all external
 # network bridges across the compute and network nodes.
 # bridge_name: Bridge name on each host for logical l2 network