From e4c4fcb2a6571c3c6c0969c092187e1f83c6c427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Jeanneret?= Date: Wed, 5 Jun 2019 17:42:29 +0200 Subject: [PATCH] Enable kill script for Neutron agents Neutron introduced "kill script" support for its agents, allowing to do more than a simple "kill ". This patch intends to activate this new feature, allowing to avoid dangling containers with failed exit state. It supports the "HUP" and "9" signal - first one invokes the "kill --signal HUP" commande from the container_cli, while the second one will stop and delete the container. Other signals will return an error, since they aren't known. The kill-script also supports the global Debug flag for a more verbose output. This patch also adds a soon to be deprecated parameter DockerAdditionalSockets in order to make the change compatible with setups still using Docker (HA deploy on Centos-7 and RHEL-7 for example). For more information about Neutron new kill script feature, please have a look at this change: I29dfbedfb7167982323dcff1c4554ee780cc48db Depends-On: https://review.opendev.org/661760 Change-Id: Iafa57b462f5ee205345a8d6e6d460ab68f312099 --- deployment/neutron/kill-script | 71 +++++++++++++++++++ .../neutron-dhcp-container-puppet.yaml | 31 ++++++++ .../neutron/neutron-l3-container-puppet.yaml | 45 ++++++++++++ .../ovn/ovn-metadata-container-puppet.yaml | 24 +++++++ .../neutron-kill-script-4d8b6a0130f6fcdd.yaml | 6 ++ 5 files changed, 177 insertions(+) create mode 100644 deployment/neutron/kill-script create mode 100644 releasenotes/notes/neutron-kill-script-4d8b6a0130f6fcdd.yaml diff --git a/deployment/neutron/kill-script b/deployment/neutron/kill-script new file mode 100644 index 0000000000..fe1147131b --- /dev/null +++ b/deployment/neutron/kill-script @@ -0,0 +1,71 @@ +#!/bin/bash +{% if debug_enabled|bool -%} +set -x +{% endif -%} +add_date() { + echo "$(date) $@" +} + +# Set up script logging for debugging purpose. +# It will be taken care of by logrotate since there is the .log +# suffix. +exec 3>&1 4>&2 +trap 'exec 2>&4 1>&3' 0 1 2 3 +exec 1>>/var/log/neutron/kill-script.log 2>&1 + +SIG=$1 +PID=$2 +NETNS=$(ip netns identify ${PID}) + +if [ "x${NETNS}" == "x" ]; then + add_date "No network namespace detected, exiting" + exit 1 +fi + +{% if container_cli == 'podman' %} +CLI="nsenter --net=/run/netns/${NETNS} --preserve-credentials -m -t 1 podman" +{% elif container_cli == 'docker' %} +{% if docker_additional_sockets and docker_additional_sockets|length > 0-%} +export DOCKER_HOST=unix://{{ docker_additional_sockets[0] }} +{% endif -%} +CLI='docker' +{% else %} +CLI='echo noop' +{% endif %} + +kill_container() { + add_date "Stopping container $1 ($2)" + $CLI stop $2 + add_date "Deleting container $1 ($2)" + $CLI rm $2 +} + +hup_container() { + add_date "Sending HUP signal to $1 ($2)" + $CLI kill --signal HUP $2 +} + +{% raw -%} +if [ -f /proc/$PID/cgroup ]; then + # Get container ID based on process cgroups + CT_ID=$(awk 'BEGIN {FS="[-.]"} /name=/{print $3}' /proc/$PID/cgroup) + CT_NAME=$($CLI inspect -f '{{.Name}}' $CT_ID) + + case $SIG in + HUP) + hup_container $CT_NAME $CT_ID + ;; + 9) + kill_container $CT_NAME $CT_ID + ;; + *) + add_date "Unknown action ${SIG} for ${$CT_NAME} ${CT_ID}" + exit 1 + ;; + esac + +else + add_date "No such PID: ${PID}" + exit 1 +fi +{% endraw %} diff --git a/deployment/neutron/neutron-dhcp-container-puppet.yaml b/deployment/neutron/neutron-dhcp-container-puppet.yaml index 224de71fc8..8c98daaf29 100644 --- a/deployment/neutron/neutron-dhcp-container-puppet.yaml +++ b/deployment/neutron/neutron-dhcp-container-puppet.yaml @@ -14,6 +14,11 @@ parameters: default: ['nofile=16384'] description: ulimit for Neutron DHCP Agent Container type: comma_delimited_list + DockerAdditionalSockets: + default: ['/var/lib/openstack/docker.sock'] + description: Additional domain sockets for the docker daemon to bind to (useful for mounting + into containers that launch other containers) + type: comma_delimited_list NeutronEnableDnsmasqDockerWrapper: description: Generate a dnsmasq wrapper script so that neutron launches dnsmasq in a separate container. @@ -285,6 +290,31 @@ outputs: - path: /etc/pki/tls/private/neutron.key owner: neutron:neutron container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]} + deploy_steps_tasks: + - when: step|int == 1 + block: + - name: set conditions + set_fact: + dnsmasq_wrapper_enabled: {get_param: NeutronEnableDnsmasqDockerWrapper} + haproxy_wrapper_enabled: {get_param: NeutronEnableHaproxyDockerWrapper} + debug_enabled: {get_param: Debug} + docker_additional_sockets: {get_param: DockerAdditionalSockets} + - name: create kill_scripts directory within /var/lib/neutron + file: + state: directory + path: /var/lib/neutron/kill_scripts + - name: create dnsmasq dhcp kill script + when: dnsmasq_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/dnsmasq-kill + mode: 0755 + content: {get_file: ./kill-script} + - name: create haproxy kill script + when: haproxy_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/haproxy-kill + mode: 0755 + content: {get_file: ./kill-script} docker_config: step_2: create_dnsmasq_wrapper: @@ -331,6 +361,7 @@ outputs: - /run/openvswitch:/run/openvswitch:shared,z - /var/lib/neutron:/var/lib/neutron:shared,z - /run/netns:/run/netns:shared + - /var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z - if: - docker_enabled diff --git a/deployment/neutron/neutron-l3-container-puppet.yaml b/deployment/neutron/neutron-l3-container-puppet.yaml index 6e07f5f024..fa2ad9cb69 100644 --- a/deployment/neutron/neutron-l3-container-puppet.yaml +++ b/deployment/neutron/neutron-l3-container-puppet.yaml @@ -14,6 +14,11 @@ parameters: default: ['nofile=16384'] description: ulimit for Neutron L3 Agent Container type: comma_delimited_list + DockerAdditionalSockets: + default: ['/var/lib/openstack/docker.sock'] + description: Additional domain sockets for the docker daemon to bind to (useful for mounting + into containers that launch other containers) + type: comma_delimited_list NeutronL3AgentLoggingSource: type: json default: @@ -244,6 +249,45 @@ outputs: owner: neutron:neutron recurse: true container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]} + deploy_steps_tasks: + - when: step|int == 1 + block: + - name: set conditions + set_fact: + keepalived_wrapper_enabled: {get_param: NeutronEnableKeepalivedWrapper} + haproxy_wrapper_enabled: {get_param: NeutronEnableHaproxyDockerWrapper} + dibbler_wrapper_enabled: {get_param: NeutronEnableDibblerDockerWrapper} + radvd_wrapper_enabled: {get_param: NeutronEnableRadvdDockerWrapper} + debug_enabled: {get_param: Debug} + docker_additional_sockets: {get_param: DockerAdditionalSockets} + - name: create kill_scripts directory within /var/lib/neutron + file: + state: directory + path: /var/lib/neutron/kill_scripts + - name: create keepalived kill script + when: keepalived_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/keepalived-kill + mode: 0755 + content: {get_file: ./kill-script} + - name: create haproxy kill script + when: haproxy_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/haproxy-kill + mode: 0755 + content: {get_file: ./kill-script} + - name: create dibbler kill script + when: dibbler_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/dibbler-kill + mode: 0755 + content: {get_file: ./kill-script} + - name: create radvd kill script + when: radvd_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/radvd-kill + mode: 0755 + content: {get_file: ./kill-script} docker_config: step_2: create_keepalived_wrapper: @@ -289,6 +333,7 @@ outputs: - /run/openvswitch:/run/openvswitch:shared,z - /var/lib/neutron:/var/lib/neutron:shared,z - /run/netns:/run/netns:shared + - /var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z - if: - docker_enabled diff --git a/deployment/ovn/ovn-metadata-container-puppet.yaml b/deployment/ovn/ovn-metadata-container-puppet.yaml index b185df5635..8d37dbaf89 100644 --- a/deployment/ovn/ovn-metadata-container-puppet.yaml +++ b/deployment/ovn/ovn-metadata-container-puppet.yaml @@ -102,6 +102,11 @@ parameters: EnableInternalTLS: type: boolean default: false + DockerAdditionalSockets: + default: ['/var/lib/openstack/docker.sock'] + description: Additional domain sockets for the docker daemon to bind to (useful for mounting + into containers that launch other containers) + type: comma_delimited_list conditions: haproxy_wrapper_enabled: {equals: [{get_param: OVNEnableHaproxyDockerWrapper}, true]} @@ -208,6 +213,24 @@ outputs: owner: neutron:neutron recurse: true container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]} + deploy_steps_tasks: + - when: step|int == 1 + block: + - name: set conditions + set_fact: + haproxy_wrapper_enabled: {get_param: OVNEnableHaproxyDockerWrapper} + debug_enabled: {get_param: Debug} + docker_additional_sockets: {get_param: DockerAdditionalSockets} + - name: create kill_scripts directory within /var/lib/neutron + file: + state: directory + path: /var/lib/neutron/kill_scripts + - name: create haproxy kill script + when: haproxy_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/haproxy-kill + mode: 0755 + content: {get_file: ../neutron/kill-script} docker_config: step_2: create_haproxy_wrapper: @@ -269,6 +292,7 @@ outputs: - /run/openvswitch:/run/openvswitch:shared,z - /var/lib/neutron:/var/lib/neutron:shared,z - /run/netns:/run/netns:shared + - /var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z - if: - docker_enabled diff --git a/releasenotes/notes/neutron-kill-script-4d8b6a0130f6fcdd.yaml b/releasenotes/notes/neutron-kill-script-4d8b6a0130f6fcdd.yaml new file mode 100644 index 0000000000..f08b02213f --- /dev/null +++ b/releasenotes/notes/neutron-kill-script-4d8b6a0130f6fcdd.yaml @@ -0,0 +1,6 @@ +--- +features: + - Enables new Neutron "kill script" feature in order to avoid dangling + containers when it kills an agent. +fixes: + - https://bugs.launchpad.net/neutron/+bug/1825943