Enable kill script for Neutron agents

Neutron introduced "kill script" support for its agents, allowing to do
more than a simple "kill <pid>".

This patch intends to activate this new feature, allowing to avoid
dangling containers with failed exit state.

It supports the "HUP" and "9" signal - first one invokes the "kill
--signal HUP" commande from the container_cli, while the second one will
stop and delete the container.
Other signals will return an error, since they aren't known.

The kill-script also supports the global Debug flag for a more verbose
output.

This patch also adds a soon to be deprecated parameter
DockerAdditionalSockets in order to make the change compatible with
setups still using Docker (HA deploy on Centos-7 and RHEL-7 for
example).

For more information about Neutron new kill script feature, please have
a look at this change: I29dfbedfb7167982323dcff1c4554ee780cc48db

Depends-On: https://review.opendev.org/661760
Change-Id: Iafa57b462f5ee205345a8d6e6d460ab68f312099

deployment/neutron/kill-script

@@ -0,0 +1,71 @@
+#!/bin/bash
+{% if debug_enabled|bool -%}
+set -x
+{% endif -%}
+add_date() {
+  echo "$(date) $@"
+}
+
+# Set up script logging for debugging purpose.
+# It will be taken care of by logrotate since there is the .log
+# suffix.
+exec 3>&1 4>&2
+trap 'exec 2>&4 1>&3' 0 1 2 3
+exec 1>>/var/log/neutron/kill-script.log 2>&1
+
+SIG=$1
+PID=$2
+NETNS=$(ip netns identify ${PID})
+
+if [ "x${NETNS}" == "x" ]; then
+  add_date "No network namespace detected, exiting"
+  exit 1
+fi
+
+{% if container_cli == 'podman' %}
+CLI="nsenter --net=/run/netns/${NETNS} --preserve-credentials -m -t 1 podman"
+{% elif container_cli == 'docker' %}
+{% if docker_additional_sockets and docker_additional_sockets|length > 0-%}
+export DOCKER_HOST=unix://{{ docker_additional_sockets[0] }}
+{% endif -%}
+CLI='docker'
+{% else %}
+CLI='echo noop'
+{% endif %}
+
+kill_container() {
+  add_date "Stopping container $1 ($2)"
+  $CLI stop $2
+  add_date "Deleting container $1 ($2)"
+  $CLI rm $2
+}
+
+hup_container() {
+  add_date "Sending HUP signal to $1 ($2)"
+  $CLI kill --signal HUP $2
+}
+
+{% raw -%}
+if [ -f /proc/$PID/cgroup ]; then
+  # Get container ID based on process cgroups
+  CT_ID=$(awk 'BEGIN {FS="[-.]"} /name=/{print $3}' /proc/$PID/cgroup)
+  CT_NAME=$($CLI inspect -f '{{.Name}}' $CT_ID)
+
+  case $SIG in
+    HUP)
+      hup_container $CT_NAME $CT_ID
+      ;;
+    9)
+      kill_container $CT_NAME $CT_ID
+      ;;
+    *)
+      add_date "Unknown action ${SIG} for ${$CT_NAME} ${CT_ID}"
+      exit 1
+      ;;
+  esac
+
+else
+  add_date "No such PID: ${PID}"
+  exit 1
+fi
+{% endraw %}

deployment/neutron/neutron-dhcp-container-puppet.yaml

@@ -14,6 +14,11 @@ parameters:
     default: ['nofile=16384']
     description: ulimit for Neutron DHCP Agent Container
     type: comma_delimited_list
+  DockerAdditionalSockets:
+    default: ['/var/lib/openstack/docker.sock']
+    description: Additional domain sockets for the docker daemon to bind to (useful for mounting
+                 into containers that launch other containers)
+    type: comma_delimited_list
   NeutronEnableDnsmasqDockerWrapper:
     description: Generate a dnsmasq wrapper script so that neutron launches
                  dnsmasq in a separate container.
@@ -285,6 +290,31 @@ outputs:
             - path: /etc/pki/tls/private/neutron.key
               owner: neutron:neutron
       container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]}
+      deploy_steps_tasks:
+        - when: step|int == 1
+          block:
+            - name: set conditions
+              set_fact:
+                dnsmasq_wrapper_enabled: {get_param: NeutronEnableDnsmasqDockerWrapper}
+                haproxy_wrapper_enabled: {get_param: NeutronEnableHaproxyDockerWrapper}
+                debug_enabled: {get_param: Debug}
+                docker_additional_sockets: {get_param: DockerAdditionalSockets}
+            - name: create kill_scripts directory within /var/lib/neutron
+              file:
+                state: directory
+                path: /var/lib/neutron/kill_scripts
+            - name: create dnsmasq dhcp kill script
+              when: dnsmasq_wrapper_enabled|bool
+              copy:
+                dest: /var/lib/neutron/kill_scripts/dnsmasq-kill
+                mode: 0755
+                content: {get_file: ./kill-script}
+            - name: create haproxy kill script
+              when: haproxy_wrapper_enabled|bool
+              copy:
+                dest: /var/lib/neutron/kill_scripts/haproxy-kill
+                mode: 0755
+                content: {get_file: ./kill-script}
       docker_config:
         step_2:
           create_dnsmasq_wrapper:
@@ -331,6 +361,7 @@ outputs:
                   - /run/openvswitch:/run/openvswitch:shared,z
                   - /var/lib/neutron:/var/lib/neutron:shared,z
                   - /run/netns:/run/netns:shared
+                  - /var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z
                 -
                   if:
                     - docker_enabled

deployment/neutron/neutron-l3-container-puppet.yaml

@@ -14,6 +14,11 @@ parameters:
     default: ['nofile=16384']
     description: ulimit for Neutron L3 Agent Container
     type: comma_delimited_list
+  DockerAdditionalSockets:
+    default: ['/var/lib/openstack/docker.sock']
+    description: Additional domain sockets for the docker daemon to bind to (useful for mounting
+                 into containers that launch other containers)
+    type: comma_delimited_list
   NeutronL3AgentLoggingSource:
     type: json
     default:
@@ -244,6 +249,45 @@ outputs:
               owner: neutron:neutron
               recurse: true
       container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]}
+      deploy_steps_tasks:
+        - when: step|int == 1
+          block:
+            - name: set conditions
+              set_fact:
+                keepalived_wrapper_enabled: {get_param: NeutronEnableKeepalivedWrapper}
+                haproxy_wrapper_enabled: {get_param: NeutronEnableHaproxyDockerWrapper}
+                dibbler_wrapper_enabled: {get_param: NeutronEnableDibblerDockerWrapper}
+                radvd_wrapper_enabled: {get_param: NeutronEnableRadvdDockerWrapper}
+                debug_enabled: {get_param: Debug}
+                docker_additional_sockets: {get_param: DockerAdditionalSockets}
+            - name: create kill_scripts directory within /var/lib/neutron
+              file:
+                state: directory
+                path: /var/lib/neutron/kill_scripts
+            - name: create keepalived kill script
+              when: keepalived_wrapper_enabled|bool
+              copy:
+                dest: /var/lib/neutron/kill_scripts/keepalived-kill
+                mode: 0755
+                content: {get_file: ./kill-script}
+            - name: create haproxy kill script
+              when: haproxy_wrapper_enabled|bool
+              copy:
+                dest: /var/lib/neutron/kill_scripts/haproxy-kill
+                mode: 0755
+                content: {get_file: ./kill-script}
+            - name: create dibbler kill script
+              when: dibbler_wrapper_enabled|bool
+              copy:
+                dest: /var/lib/neutron/kill_scripts/dibbler-kill
+                mode: 0755
+                content: {get_file: ./kill-script}
+            - name: create radvd kill script
+              when: radvd_wrapper_enabled|bool
+              copy:
+                dest: /var/lib/neutron/kill_scripts/radvd-kill
+                mode: 0755
+                content: {get_file: ./kill-script}
       docker_config:
         step_2:
           create_keepalived_wrapper:
@@ -289,6 +333,7 @@ outputs:
                   - /run/openvswitch:/run/openvswitch:shared,z
                   - /var/lib/neutron:/var/lib/neutron:shared,z
                   - /run/netns:/run/netns:shared
+                  - /var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z
                 -
                   if:
                     - docker_enabled

deployment/ovn/ovn-metadata-container-puppet.yaml

@@ -102,6 +102,11 @@ parameters:
   EnableInternalTLS:
     type: boolean
     default: false
+  DockerAdditionalSockets:
+    default: ['/var/lib/openstack/docker.sock']
+    description: Additional domain sockets for the docker daemon to bind to (useful for mounting
+                 into containers that launch other containers)
+    type: comma_delimited_list
 
 conditions:
   haproxy_wrapper_enabled: {equals: [{get_param: OVNEnableHaproxyDockerWrapper}, true]}
@@ -208,6 +213,24 @@ outputs:
               owner: neutron:neutron
               recurse: true
       container_config_scripts: {get_attr: [ContainersCommon, container_config_scripts]}
+      deploy_steps_tasks:
+        - when: step|int == 1
+          block:
+            - name: set conditions
+              set_fact:
+                haproxy_wrapper_enabled: {get_param: OVNEnableHaproxyDockerWrapper}
+                debug_enabled: {get_param: Debug}
+                docker_additional_sockets: {get_param: DockerAdditionalSockets}
+            - name: create kill_scripts directory within /var/lib/neutron
+              file:
+                state: directory
+                path: /var/lib/neutron/kill_scripts
+            - name: create haproxy kill script
+              when: haproxy_wrapper_enabled|bool
+              copy:
+                dest: /var/lib/neutron/kill_scripts/haproxy-kill
+                mode: 0755
+                content: {get_file: ../neutron/kill-script}
       docker_config:
         step_2:
           create_haproxy_wrapper:
@@ -269,6 +292,7 @@ outputs:
                   - /run/openvswitch:/run/openvswitch:shared,z
                   - /var/lib/neutron:/var/lib/neutron:shared,z
                   - /run/netns:/run/netns:shared
+                  - /var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z
                 -
                   if:
                     - docker_enabled

releasenotes/notes/neutron-kill-script-4d8b6a0130f6fcdd.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - Enables new Neutron "kill script" feature in order to avoid dangling
+    containers when it kills an agent.
+fixes:
+  - https://bugs.launchpad.net/neutron/+bug/1825943