From 62e529ce32d34f43651f6951140cd04c2b1f8043 Mon Sep 17 00:00:00 2001 From: Alex Schultz Date: Fri, 1 May 2020 07:43:54 -0600 Subject: [PATCH] Revert systemd sidecars We've found that the systemd sidecars tend to drop events when spawning multiple processes at once. Rather than continue to try and patch it, we need to go back to the drawing board. This change reverts the various patches that were related to the systemd side car code. Revert "Use exec when spawning any neutron sidecar container" This reverts commit 5b799136facc15d4e69bcede52b60d39a4a02464. Revert "Remove neutron wrappers usage" This reverts commit f4f3045c413e7da083dbd8495ef758c2ac86870d. Revert "Use a systemd service to handle sidecar containers" This reverts commit 2dc7066b050ecf22dc9e5909061272ffe765ebfc. Change-Id: I8b9578b7c7d6bd23f0b677f64afae7be76ddcadf --- deployment/neutron/kill-script | 71 +++++++++++ .../neutron-dhcp-container-puppet.yaml | 82 ++++++++----- .../neutron/neutron-l3-container-puppet.yaml | 110 ++++++++++++------ .../ovn/ovn-metadata-container-puppet.yaml | 68 +++++++---- 4 files changed, 241 insertions(+), 90 deletions(-) create mode 100644 deployment/neutron/kill-script diff --git a/deployment/neutron/kill-script b/deployment/neutron/kill-script new file mode 100644 index 0000000000..fe1147131b --- /dev/null +++ b/deployment/neutron/kill-script @@ -0,0 +1,71 @@ +#!/bin/bash +{% if debug_enabled|bool -%} +set -x +{% endif -%} +add_date() { + echo "$(date) $@" +} + +# Set up script logging for debugging purpose. +# It will be taken care of by logrotate since there is the .log +# suffix. +exec 3>&1 4>&2 +trap 'exec 2>&4 1>&3' 0 1 2 3 +exec 1>>/var/log/neutron/kill-script.log 2>&1 + +SIG=$1 +PID=$2 +NETNS=$(ip netns identify ${PID}) + +if [ "x${NETNS}" == "x" ]; then + add_date "No network namespace detected, exiting" + exit 1 +fi + +{% if container_cli == 'podman' %} +CLI="nsenter --net=/run/netns/${NETNS} --preserve-credentials -m -t 1 podman" +{% elif container_cli == 'docker' %} +{% if docker_additional_sockets and docker_additional_sockets|length > 0-%} +export DOCKER_HOST=unix://{{ docker_additional_sockets[0] }} +{% endif -%} +CLI='docker' +{% else %} +CLI='echo noop' +{% endif %} + +kill_container() { + add_date "Stopping container $1 ($2)" + $CLI stop $2 + add_date "Deleting container $1 ($2)" + $CLI rm $2 +} + +hup_container() { + add_date "Sending HUP signal to $1 ($2)" + $CLI kill --signal HUP $2 +} + +{% raw -%} +if [ -f /proc/$PID/cgroup ]; then + # Get container ID based on process cgroups + CT_ID=$(awk 'BEGIN {FS="[-.]"} /name=/{print $3}' /proc/$PID/cgroup) + CT_NAME=$($CLI inspect -f '{{.Name}}' $CT_ID) + + case $SIG in + HUP) + hup_container $CT_NAME $CT_ID + ;; + 9) + kill_container $CT_NAME $CT_ID + ;; + *) + add_date "Unknown action ${SIG} for ${$CT_NAME} ${CT_ID}" + exit 1 + ;; + esac + +else + add_date "No such PID: ${PID}" + exit 1 +fi +{% endraw %} diff --git a/deployment/neutron/neutron-dhcp-container-puppet.yaml b/deployment/neutron/neutron-dhcp-container-puppet.yaml index d4506884f8..3a3db31526 100644 --- a/deployment/neutron/neutron-dhcp-container-puppet.yaml +++ b/deployment/neutron/neutron-dhcp-container-puppet.yaml @@ -215,7 +215,19 @@ outputs: map_merge: - get_attr: [NeutronBase, role_data, config_settings] - get_attr: [NeutronLogging, config_settings] - - neutron::agents::dhcp::enable_isolated_metadata: {get_param: NeutronEnableIsolatedMetadata} + - tripleo::profile::base::neutron::dhcp_agent_wrappers::enable_dnsmasq_wrapper: {get_param: NeutronEnableDnsmasqDockerWrapper} + tripleo::profile::base::neutron::dhcp_agent_wrappers::dnsmasq_process_wrapper: '/var/lib/neutron/dnsmasq_wrapper' + tripleo::profile::base::neutron::dhcp_agent_wrappers::dnsmasq_image: {get_param: ContainerNeutronDHCPImage} + tripleo::profile::base::neutron::dhcp_agent_wrappers::enable_haproxy_wrapper: {get_param: NeutronEnableHaproxyDockerWrapper} + tripleo::profile::base::neutron::dhcp_agent_wrappers::haproxy_process_wrapper: '/var/lib/neutron/dhcp_haproxy_wrapper' + tripleo::profile::base::neutron::dhcp_agent_wrappers::haproxy_image: {get_param: ContainerNeutronDHCPImage} + tripleo::profile::base::neutron::dhcp_agent_wrappers::debug: + if: + - service_wrapper_debug_unset + - {get_param: Debug } + - {get_param: NeutronWrapperDebug} + tripleo::profile::base::neutron::container_cli: {get_param: ContainerCli} + neutron::agents::dhcp::enable_isolated_metadata: {get_param: NeutronEnableIsolatedMetadata} neutron::agents::dhcp::enable_force_metadata: {get_param: NeutronEnableForceMetadata} neutron::agents::dhcp::enable_metadata_network: {get_param: NeutronEnableMetadataNetwork} neutron::agents::dhcp::dnsmasq_local_resolv: {get_param: NeutronEnableInternalDNS} @@ -311,33 +323,45 @@ outputs: - service_debug_unset - {get_param: Debug } - {get_param: NeutronWrapperDebug} - - name: Create dhcp systemd wrappers - include_role: - name: tripleo_systemd_wrapper - vars: - tripleo_systemd_wrapper_cmd: "{{ dhcp_wrapper_item.cmd }}" - tripleo_systemd_wrapper_config_bind_mount: "/var/lib/config-data/puppet-generated/neutron:/etc/neutron:ro" - tripleo_systemd_wrapper_container_cli: "{{ container_cli }}" - tripleo_systemd_wrapper_debug: "{{ debug_enabled }}" - tripleo_systemd_wrapper_docker_additional_sockets: {get_param: DockerAdditionalSockets} - tripleo_systemd_wrapper_image_name: {get_param: ContainerNeutronDHCPImage} - tripleo_systemd_wrapper_service_dir: /var/lib/neutron - tripleo_systemd_wrapper_service_kill_script: "{{ dhcp_wrapper_item.kill_script }}" - tripleo_systemd_wrapper_service_name: "{{ dhcp_wrapper_item.name }}" - loop_control: - loop_var: dhcp_wrapper_item - loop: - - name: dhcp_dnsmasq - cmd: exec /usr/sbin/dnsmasq -k - kill_script: dnsmasq-kill - - name: dhcp_haproxy - cmd: >- - $(if [ -f /usr/sbin/haproxy-systemd-wrapper ]; then - echo "exec /usr/sbin/haproxy -Ds"; - else - echo "exec /usr/sbin/haproxy -Ws"; fi) - kill_script: haproxy-kill + docker_additional_sockets: {get_param: DockerAdditionalSockets} + - name: create kill_scripts directory within /var/lib/neutron + file: + state: directory + path: /var/lib/neutron/kill_scripts + - name: create dnsmasq dhcp kill script + when: dnsmasq_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/dnsmasq-kill + mode: 0755 + content: {get_file: ./kill-script} + - name: create haproxy kill script + when: haproxy_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/haproxy-kill + mode: 0755 + content: {get_file: ./kill-script} docker_config: + step_2: + create_dnsmasq_wrapper: + start_order: 1 + detach: false + net: host + pid: host + user: root + command: # '/container_puppet_apply.sh "STEP" "TAGS" "CONFIG" "DEBUG"' + list_concat: + - + - '/container_puppet_apply.sh' + - '4' + - 'file' + - 'include ::tripleo::profile::base::neutron::dhcp_agent_wrappers' + image: {get_param: ContainerNeutronDHCPImage} + volumes: + list_concat: + - {get_attr: [ContainersCommon, container_puppet_apply_volumes]} + - + - /run/openvswitch:/run/openvswitch:shared,z + - /var/lib/neutron:/var/lib/neutron:shared,z step_4: neutron_dhcp: start_order: 10 @@ -371,12 +395,12 @@ outputs: - if: - dnsmasq_wrapper_enabled - - - /var/lib/neutron/dhcp_dnsmasq/wrapper:/usr/local/bin/dnsmasq:ro + - - /var/lib/neutron/dnsmasq_wrapper:/usr/local/bin/dnsmasq:ro - null - if: - haproxy_wrapper_enabled - - - /var/lib/neutron/dhcp_haproxy/wrapper:/usr/local/bin/haproxy:ro + - - /var/lib/neutron/dhcp_haproxy_wrapper:/usr/local/bin/haproxy:ro - null - if: diff --git a/deployment/neutron/neutron-l3-container-puppet.yaml b/deployment/neutron/neutron-l3-container-puppet.yaml index 9dc31bcbf7..1db34576b8 100644 --- a/deployment/neutron/neutron-l3-container-puppet.yaml +++ b/deployment/neutron/neutron-l3-container-puppet.yaml @@ -188,6 +188,26 @@ outputs: - get_attr: [NeutronBase, role_data, config_settings] - get_attr: [RoleParametersValue, value] - get_attr: [NeutronLogging, config_settings] + - tripleo::profile::base::neutron::l3_agent_wrappers::enable_keepalived_wrapper: {get_param: NeutronEnableKeepalivedWrapper} + tripleo::profile::base::neutron::l3_agent_wrappers::keepalived_process_wrapper: '/var/lib/neutron/keepalived_wrapper' + # TODO(beagles): this can be removed after a cleanup of the related puppet-tripleo code. + tripleo::profile::base::neutron::l3_agent_wrappers::keepalived_state_change_wrapper: '/var/lib/neutron/keepalived_state_change_wrapper' + tripleo::profile::base::neutron::l3_agent_wrappers::keepalived_image: {get_param: ContainerNeutronL3AgentImage} + tripleo::profile::base::neutron::l3_agent_wrappers::enable_haproxy_wrapper: {get_param: NeutronEnableHaproxyDockerWrapper} + tripleo::profile::base::neutron::l3_agent_wrappers::haproxy_process_wrapper: '/var/lib/neutron/l3_haproxy_wrapper' + tripleo::profile::base::neutron::l3_agent_wrappers::haproxy_image: {get_param: ContainerNeutronL3AgentImage} + tripleo::profile::base::neutron::l3_agent_wrappers::enable_dibbler_wrapper: {get_param: NeutronEnableDibblerDockerWrapper} + tripleo::profile::base::neutron::l3_agent_wrappers::dibbler_process_wrapper: '/var/lib/neutron/dibbler_wrapper' + tripleo::profile::base::neutron::l3_agent_wrappers::dibbler_image: {get_param: ContainerNeutronL3AgentImage} + tripleo::profile::base::neutron::l3_agent_wrappers::enable_radvd_wrapper: {get_param: NeutronEnableRadvdDockerWrapper} + tripleo::profile::base::neutron::l3_agent_wrappers::radvd_process_wrapper: '/var/lib/neutron/radvd_wrapper' + tripleo::profile::base::neutron::l3_agent_wrappers::radvd_image: {get_param: ContainerNeutronL3AgentImage} + tripleo::profile::base::neutron::l3_agent_wrappers::debug: + if: + - service_debug_unset + - {get_param: Debug } + - {get_param: NeutronWrapperDebug} + tripleo::profile::base::neutron::container_cli: {get_param: ContainerCli} - neutron::agents::l3::debug: if: - service_debug_unset @@ -249,39 +269,57 @@ outputs: - service_debug_unset - {get_param: Debug } - {get_param: NeutronWrapperDebug} - - name: Create l3 systemd wrappers - include_role: - name: tripleo_systemd_wrapper - vars: - tripleo_systemd_wrapper_cmd: "{{ l3_wrapper_item.cmd }}" - tripleo_systemd_wrapper_config_bind_mount: "/var/lib/config-data/puppet-generated/neutron:/etc/neutron:ro" - tripleo_systemd_wrapper_container_cli: "{{ container_cli }}" - tripleo_systemd_wrapper_debug: "{{ debug_enabled }}" - tripleo_systemd_wrapper_docker_additional_sockets: {get_param: DockerAdditionalSockets} - tripleo_systemd_wrapper_image_name: {get_param: ContainerNeutronL3AgentImage} - tripleo_systemd_wrapper_service_dir: /var/lib/neutron - tripleo_systemd_wrapper_service_kill_script: "{{ l3_wrapper_item.kill_script }}" - tripleo_systemd_wrapper_service_name: "{{ l3_wrapper_item.name }}" - loop_control: - loop_var: l3_wrapper_item - loop: - - name: l3_keepalived - cmd: exec /usr/sbin/keepalived -n -l -D - kill_script: keepalived-kill - - name: l3_haproxy - cmd: >- - $(if [ -f /usr/sbin/haproxy-systemd-wrapper ]; then - echo "exec /usr/sbin/haproxy -Ds"; - else - echo "exec /usr/sbin/haproxy -Ws"; fi) - kill_script: haproxy-kill - - name: l3_dibbler - cmd: exec /usr/sbin/dibbler-client run - kill_script: dibbler-kill - - name: l3_radvd - cmd: exec /usr/sbin/radvd -n - kill_script: radvd-kill + docker_additional_sockets: {get_param: DockerAdditionalSockets} + - name: create kill_scripts directory within /var/lib/neutron + file: + state: directory + path: /var/lib/neutron/kill_scripts + - name: create keepalived kill script + when: keepalived_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/keepalived-kill + mode: 0755 + content: {get_file: ./kill-script} + - name: create haproxy kill script + when: haproxy_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/haproxy-kill + mode: 0755 + content: {get_file: ./kill-script} + - name: create dibbler kill script + when: dibbler_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/dibbler-kill + mode: 0755 + content: {get_file: ./kill-script} + - name: create radvd kill script + when: radvd_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/radvd-kill + mode: 0755 + content: {get_file: ./kill-script} docker_config: + step_2: + create_keepalived_wrapper: + start_order: 1 + detach: false + net: host + pid: host + user: root + command: # '/container_puppet_apply.sh "STEP" "TAGS" "CONFIG" "DEBUG"' + list_concat: + - + - '/container_puppet_apply.sh' + - '4' + - 'file' + - 'include ::tripleo::profile::base::neutron::l3_agent_wrappers' + image: {get_param: ContainerNeutronL3AgentImage} + volumes: + list_concat: + - {get_attr: [ContainersCommon, container_puppet_apply_volumes]} + - + - /run/openvswitch:/run/openvswitch:shared,z + - /var/lib/neutron:/var/lib/neutron:shared,z step_4: neutron_l3_agent: start_order: 10 @@ -314,22 +352,22 @@ outputs: - if: - keepalived_wrapper_enabled - - - /var/lib/neutron/l3_keepalived/wrapper:/usr/local/bin/keepalived:ro + - - /var/lib/neutron/keepalived_wrapper:/usr/local/bin/keepalived:ro - null - if: - haproxy_wrapper_enabled - - - /var/lib/neutron/l3_haproxy/wrapper:/usr/local/bin/haproxy:ro + - - /var/lib/neutron/l3_haproxy_wrapper:/usr/local/bin/haproxy:ro - null - if: - radvd_wrapper_enabled - - - /var/lib/neutron/l3_radvd/wrapper:/usr/local/bin/radvd:ro + - - /var/lib/neutron/radvd_wrapper:/usr/local/bin/radvd:ro - null - if: - dibbler_wrapper_enabled - - - /var/lib/neutron/l3_dibbler/wrapper:/usr/local/bin/dibbler_client:ro + - - /var/lib/neutron/dibbler_wrapper:/usr/local/bin/dibbler_client:ro - null environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS diff --git a/deployment/ovn/ovn-metadata-container-puppet.yaml b/deployment/ovn/ovn-metadata-container-puppet.yaml index fe3db5fb28..302b616c42 100644 --- a/deployment/ovn/ovn-metadata-container-puppet.yaml +++ b/deployment/ovn/ovn-metadata-container-puppet.yaml @@ -160,7 +160,16 @@ outputs: map_merge: - get_attr: [NeutronBase, role_data, config_settings] - get_attr: [NeutronLogging, config_settings] - - neutron::agents::ovn_metadata::shared_secret: {get_param: NeutronMetadataProxySharedSecret} + - tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::enable_haproxy_wrapper: {get_param: OVNEnableHaproxyDockerWrapper} + tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::haproxy_process_wrapper: '/var/lib/neutron/ovn_metadata_haproxy_wrapper' + tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::haproxy_image: {get_param: ContainerOvnMetadataImage} + tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::debug: + if: + - service_debug_unset + - {get_param: Debug } + - {get_param: OVNWrapperDebug} + tripleo::profile::base::neutron::container_cli: {get_param: ContainerCli} + neutron::agents::ovn_metadata::shared_secret: {get_param: NeutronMetadataProxySharedSecret} neutron::agents::ovn_metadata::auth_password: {get_param: NeutronPassword} neutron::agents::ovn_metadata::auth_url: { get_param: [EndpointMap, KeystoneInternal, uri_no_suffix] } neutron::agents::ovn_metadata::auth_tenant: 'service' @@ -253,30 +262,39 @@ outputs: - service_debug_unset - {get_param: Debug } - {get_param: NeutronWrapperDebug} - - name: Create ovn metadata systemd wrappers - include_role: - name: tripleo_systemd_wrapper - vars: - tripleo_systemd_wrapper_cmd: "{{ ovn_wrapper_item.cmd }}" - tripleo_systemd_wrapper_config_bind_mount: "/var/lib/config-data/puppet-generated/neutron:/etc/neutron:ro" - tripleo_systemd_wrapper_container_cli: "{{ container_cli }}" - tripleo_systemd_wrapper_debug: "{{ debug_enabled }}" - tripleo_systemd_wrapper_docker_additional_sockets: {get_param: DockerAdditionalSockets} - tripleo_systemd_wrapper_image_name: {get_param: ContainerOvnMetadataImage} - tripleo_systemd_wrapper_service_dir: /var/lib/neutron - tripleo_systemd_wrapper_service_kill_script: "{{ ovn_wrapper_item.kill_script }}" - tripleo_systemd_wrapper_service_name: "{{ ovn_wrapper_item.name }}" - loop_control: - loop_var: ovn_wrapper_item - loop: - - name: ovn_metadata_haproxy - cmd: >- - $(if [ -f /usr/sbin/haproxy-systemd-wrapper ]; then - echo "exec /usr/sbin/haproxy -Ds"; - else - echo "exec /usr/sbin/haproxy -Ws"; fi) - kill_script: haproxy-kill + docker_additional_sockets: {get_param: DockerAdditionalSockets} + - name: create kill_scripts directory within /var/lib/neutron + file: + state: directory + path: /var/lib/neutron/kill_scripts + - name: create haproxy kill script + when: haproxy_wrapper_enabled|bool + copy: + dest: /var/lib/neutron/kill_scripts/haproxy-kill + mode: 0755 + content: {get_file: ../neutron/kill-script} docker_config: + step_2: + create_haproxy_wrapper: + start_order: 1 + detach: false + net: host + pid: host + user: root + command: # '/container_puppet_apply.sh "STEP" "TAGS" "CONFIG" "DEBUG"' + list_concat: + - + - '/container_puppet_apply.sh' + - '4' + - 'file' + - 'include ::tripleo::profile::base::neutron::ovn_metadata_agent_wrappers' + image: {get_param: ContainerOvnMetadataImage} + volumes: + list_concat: + - {get_attr: [ContainersCommon, container_puppet_apply_volumes]} + - + - /run/openvswitch:/run/openvswitch:shared,z + - /var/lib/neutron:/var/lib/neutron:shared,z step_4: setup_ovs_manager: start_order: 0 @@ -327,7 +345,7 @@ outputs: - if: - haproxy_wrapper_enabled - - - /var/lib/neutron/ovn_metadata_haproxy/wrapper:/usr/local/bin/haproxy:ro + - - /var/lib/neutron/ovn_metadata_haproxy_wrapper:/usr/local/bin/haproxy:ro - null - if: - internal_tls_enabled