diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh index 9eac6d61..c7a82daa 100755 --- a/devstack-vm-gate-wrap.sh +++ b/devstack-vm-gate-wrap.sh @@ -400,6 +400,27 @@ export DEVSTACK_GATE_TOPOLOGY=${DEVSTACK_GATE_TOPOLOGY:-aio} # for jobs that know exactly which repos they need. export DEVSTACK_GATE_PROJECTS_OVERRIDE=${DEVSTACK_GATE_PROJECTS_OVERRIDE:-""} +# Set this to enable remote logging of the console via UDP packets to +# a specified ipv4 ip:port (note; not hostname -- ip address only). +# This can be extremely useful if a host is oopsing or dropping off +# the network amd you are not getting any useful logs from jenkins. +# +# To capture these logs, enable a netcat/socat type listener to +# capture UDP packets at the specified remote ip. For example: +# +# $ nc -v -u -l -p 6666 | tee save-output.log +# or +# $ socat udp-recv:6666 - | tee save-output.log +# +# One further trick is to send interesting data to /dev/ksmg; this +# data will get out over the netconsole even if the main interfaces +# have been disabled, etc. e.g. +# +# $ ip addr | sudo tee /dev/ksmg +# +export DEVSTACK_GATE_NETCONSOLE=${DEVSTACK_GATE_NETCONSOLE:-""} +enable_netconsole + if [ -n "$DEVSTACK_GATE_PROJECTS_OVERRIDE" ]; then PROJECTS=$DEVSTACK_GATE_PROJECTS_OVERRIDE fi diff --git a/functions.sh b/functions.sh index f8f0643b..317ab53f 100644 --- a/functions.sh +++ b/functions.sh @@ -864,6 +864,45 @@ function remote_copy_file { scp $ssh_opts "$src" "$dest" } +# enable_netconsole +function enable_netconsole { + + # do nothing if not set + if [[ $DEVSTACK_GATE_NETCONSOLE = "" ]]; then + return + fi + + local remote_ip=$(echo $DEVSTACK_GATE_NETCONSOLE | awk -F: -e '{print $1}') + local remote_port=$(echo $DEVSTACK_GATE_NETCONSOLE | awk -F: -e '{print $2}') + + # netconsole requires the device to send and the destitation MAC, + # which is obviously on the same subnet. The way to get packets + # out to the world is specify the default gw as the remote + # destination. + local default_gw=$(ip route | grep default | awk '{print $3}') + local gw_mac=$(arp $default_gw | grep $default_gw | awk '{print $3}') + local gw_dev=$(ip route | grep default | awk '{print $5}') + + # turn up message output + sudo dmesg -n 8 + + sudo modprobe configfs + sudo modprobe netconsole + + sudo mount none -t configfs /sys/kernel/config + + sudo mkdir /sys/kernel/config/netconsole/target1 + + pushd /sys/kernel/config/netconsole/target1 + echo "$gw_dev" | sudo tee ./dev_name + echo "$remote_ip" | sudo tee ./remote_ip + echo "$gw_mac" | sudo tee ./remote_mac + echo "$remote_port" | sudo tee ./remote_port + echo 1 | sudo tee ./enabled + popd +} + + # This function creates an internal gre bridge to connect all external # network bridges across the compute and network nodes. # bridge_name: Bridge name on each host for logical l2 network