diff --git a/doc/source/general-roles.rst b/doc/source/general-roles.rst index ba1dff296..577486a14 100644 --- a/doc/source/general-roles.rst +++ b/doc/source/general-roles.rst @@ -13,6 +13,7 @@ General Purpose Roles .. zuul:autorole:: download-artifact .. zuul:autorole:: dstat-graph .. zuul:autorole:: emit-job-header +.. zuul:autorole:: enable-netconsole .. zuul:autorole:: ensure-dstat-graph .. zuul:autorole:: git-prepare-nodecache .. zuul:autorole:: log-inventory diff --git a/roles/enable-netconsole/README.rst b/roles/enable-netconsole/README.rst new file mode 100644 index 000000000..194d88578 --- /dev/null +++ b/roles/enable-netconsole/README.rst @@ -0,0 +1,32 @@ +Enable netconsole for host + +This enables the netconsole on a host to send kernel/dmesg logs to a +remote host. This can be very useful if a node is experiencing a +kernel oops or another form of unexpected disconnect where you can not +retrieve information via standard logging methods. + +The ``netconsole_remote_ip`` and ``netconsole_remote_port`` variables +must be set. This host can capture the logs with a command like:: + + $ nc -v -u -l -p 6666 | tee console-output.log + +or:: + + $ socat udp-recv:6666 - | tee console-output.log + +One further trick is to send interesting data to ``/dev/kmsg``, this +should make it across the netconsole even if the main interface has +been disabled, etc. e.g.:: + + $ ip addr | sudo tee /dev/kmsg + + +**Role Variables** + +.. zuul:rolevar:: netconsole_remote_ip + + The IP address of the remote host to send to. + +.. zuul:rolevar:: netconsole_remote_port + + The port listening on the remote host. diff --git a/roles/enable-netconsole/tasks/main.yaml b/roles/enable-netconsole/tasks/main.yaml new file mode 100644 index 000000000..e46e064f6 --- /dev/null +++ b/roles/enable-netconsole/tasks/main.yaml @@ -0,0 +1,82 @@ +- name: Check remote IP set + assert: + that: + - netconsole_remote_ip is defined + - netconsole_remote_port is defined + fail_msg: "Must set remote host and port" + +- name: Register netconsole target var + set_fact: + nc_target: '/sys/kernel/config/netconsole/target1' + +- name: Everything needs root + become: yes + block: + + # netconsole requires the device to send packtes from and the + # destination MAC. This works for hosts on the same subnet, but the + # way to get packets out to the world is to specify the default gw as + # the remote destination. + + - name: Get default gateway + shell: "ip route | grep default | awk '{print $3}'" + register: default_gw_cmd + + - name: Save default gateway + set_fact: + default_gw: '{{ default_gw_cmd.stdout }}' + + - name: Get default gateway MAC + shell: "arp {{ default_gw }} | grep {{ default_gw }} | awk '{print $3}'" + register: default_gw_cmd_mac + + - name: Save default gateway MAC + set_fact: + default_gw_mac: '{{ default_gw_cmd_mac.stdout }}' + + - name: Get default device + shell: "ip route | grep default | awk '{print $5}'" + register: default_gw_cmd_dev + + - name: Save default device + set_fact: + default_gw_dev: '{{ default_gw_cmd_dev.stdout }}' + + - name: Install configfs + command: modprobe configfs + + - name: Install netconsole + command: modprobe netconsole + + - name: Mount configfs + mount: + path: /sys/kernel/config + fstype: configfs + src: configfs + state: mounted + + - name: Make netconsole target directory + file: + path: '{{ nc_target }}' + state: directory + + - name: Configure gateway + shell: 'echo {{ default_gw_dev }} > {{ nc_target }}/dev_name' + + - name: Configure gateway MAC + shell: 'echo {{ default_gw_mac }} > {{ nc_target }}/remote_mac' + + - name: Configure remote IP + shell: 'echo {{ netconsole_remote_ip }} > {{ nc_target }}/remote_ip' + + - name: Configure remote port + shell: 'echo {{ netconsole_remote_port }} > {{ nc_target }}/remote_port' + + - name: Enable netconsole + shell: 'echo 1 > {{ nc_target }}/enabled' + + - name: Turn up dmesg + command: dmesg -n 8 + + - name: Send a test message + shell: "echo 'netconsole enabled' > /dev/kmsg" diff --git a/zuul-tests.d/general-roles-jobs.yaml b/zuul-tests.d/general-roles-jobs.yaml index 8bc94a9e9..891b85a29 100644 --- a/zuul-tests.d/general-roles-jobs.yaml +++ b/zuul-tests.d/general-roles-jobs.yaml @@ -232,6 +232,17 @@ vars: role_name: clear-firewall +- job: + name: zuul-jobs-test-netconsole + description: Test the netconsole setup role + files: + - ^roles/enable-netconsole/.* + run: test-playbooks/simple-role-test.yaml + vars: + role_name: enable-netconsole + netconsole_remote_ip: 192.0.2.1 + netconsole_remote_port: 6666 + - job: name: zuul-jobs-test-dstat-graph description: Test the dstat-graph roles @@ -464,6 +475,7 @@ - zuul-jobs-test-base-roles-ubuntu-xenial - zuul-jobs-test-bindep - zuul-jobs-test-clear-firewall + - zuul-jobs-test-netconsole - zuul-jobs-test-dstat-graph - zuul-jobs-test-multinode-roles-centos-7 - zuul-jobs-test-multinode-roles-debian-stretch