From 815ece74543b618bb901864a323cc6f89672022a Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Mon, 27 Nov 2017 10:56:32 -0600 Subject: [PATCH] Unify container network interfaces with networkd Unify container network interfaces using Systemd Networkd for ubuntu, centos, and openSUSE. This change allows the role to use a single way to configure container networks. Care has been taken to ensure we're able to cleanly upgrade to the new capabilities within existing environments without breaking any feature compatibility or causing any container restarts. It's also worth noting that all of the pre/post networking up/down script options have been converted to systemd "oneshot" services. This retains the ability to run adhoc scripts post network availability while also opening up this capability, which used to be ubuntu only, to all of our supported operating systems. > Our usage of `lxc-attach` was removed in favor of `nsenter` to fix a issue where multiple `lxc-attach` commands issued to a single physical host could result in a hang. > Scripts that were being generated inline have been placed into template files. This solves a long standing memory consumption issue when creating lots of containers. The old shell tasks will now be executed from a generated script. While this should also help with debugging, the main driver is to ensure better system stability. > A lot of cleanup has been done throughout the task files and templates. In the process of updating the role to use unified networking a lot of duplicate tasks, scripts, and processes have consolidated. > Handlers have been added for network connection wait conditions and to various service restarts. > The OSA plugins have been added to this role as a dependency. We rely on the connection plugins throughout the stack however we were doing a lot of workarounds to cater to the possibility of a deployer running this role without them. This change simply adds the plugins as a known dependency which allows for a more streamlined setup. Change-Id: I5d3ddcfa11d575648a69a04f2fb30236c2c89da3 Signed-off-by: Kevin Carter --- defaults/main.yml | 19 +- files/lxc-veth-wiring.sh | 29 +- handlers/main.yml | 62 +++- meta/main.yml | 3 +- ...ing-convert-networkd-5b514e604df7c429.yaml | 9 + tasks/lxc_container_config.yml | 311 ++++++++++-------- tasks/lxc_container_create.yml | 22 +- tasks/lxc_container_network_new.yml | 70 ++++ templates/container-first-run.sh.j2 | 34 ++ templates/container-setup.sh.j2 | 3 + templates/container_mac_generation.sh.j2 | 28 ++ templates/container_network.network.j2 | 37 +++ templates/debian-interface.cfg.j2 | 33 -- templates/post-up-down.oneshot.j2 | 17 + templates/pre-up-down.oneshot.j2 | 19 ++ templates/rhel-interface.j2 | 18 - templates/rhel-routes.j2 | 3 - templates/suse-interface.j2 | 11 - templates/suse-routes.j2 | 7 - templates/systemd-hostnamed.conf | 3 + templates/veth-cleanup.sh.j2 | 12 +- tests/ansible-role-requirements.yml | 4 + tests/test-containers-create.yml | 36 ++ tests/test.yml | 10 +- vars/main.yml | 19 ++ vars/redhat-7.yml | 24 +- vars/suse-42.yml | 25 ++ vars/ubuntu-16.04.yml | 29 +- 28 files changed, 617 insertions(+), 280 deletions(-) create mode 100644 releasenotes/notes/legacy-networking-convert-networkd-5b514e604df7c429.yaml create mode 100644 tasks/lxc_container_network_new.yml create mode 100644 templates/container-first-run.sh.j2 create mode 100644 templates/container_mac_generation.sh.j2 create mode 100644 templates/container_network.network.j2 delete mode 100644 templates/debian-interface.cfg.j2 create mode 100644 templates/post-up-down.oneshot.j2 create mode 100644 templates/pre-up-down.oneshot.j2 delete mode 100644 templates/rhel-interface.j2 delete mode 100644 templates/rhel-routes.j2 delete mode 100644 templates/suse-interface.j2 delete mode 100644 templates/suse-routes.j2 create mode 100644 templates/systemd-hostnamed.conf create mode 100644 tests/test-containers-create.yml create mode 100644 vars/main.yml diff --git a/defaults/main.yml b/defaults/main.yml index 95e8092..793bae9 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -37,6 +37,7 @@ lxc_container_default_config_list: - "lxc.hook.autodev=/var/lib/lxc/{{ inventory_hostname }}/autodev" lxc_container_config_list: [] lxc_container_commands: "" +lxc_container_extra_commands: "{{ _lxc_container_extra_commands | default('echo noop') }}" # A list of bind mounts to configure for the container, for example: # lxc_container_bind_mounts: @@ -89,9 +90,8 @@ lxc_container_backing_store: dir # using the directory backing. lxc_container_vg_name: lxc -# Scripts allowing the configuration of pre/post-up/down scripts in Ubuntu -# interface files. These are merged with per-interface scripts defined in the -# container_networks dict +# Scripts allowing the configuration of pre/post-up/down scripts. +# These are run as one shot services before or after networking is available. lxc_container_default_preup: [] lxc_container_default_postup: [] lxc_container_default_predown: [] @@ -147,8 +147,21 @@ lxc_container_allow_restarts: yes lxc_container_network_veth_pair_prefix: "{{ inventory_hostname[-8:].replace('-', '').replace('_', '') }}" lxc_container_network_veth_pair: "{{ lxc_container_network_veth_pair_prefix }}_{{ item.value.interface }}" +# A default set of container networks used within the LXC containers. +lxc_container_networks: + lxcbr0_address: + bridge: lxcbr0 + interface: eth0 + type: veth + # Enable fixed mac address generation for an lxc container lxc_container_fixed_mac: false # Enable destroying then recreating containers lxc_container_recreate: false + +# Enable running the veth wiring script +lxc_container_veth_wiring: false + +# Enable systemd-resolved +lxc_container_enable_resolved: true diff --git a/files/lxc-veth-wiring.sh b/files/lxc-veth-wiring.sh index 8e6bb17..38f4fdc 100644 --- a/files/lxc-veth-wiring.sh +++ b/files/lxc-veth-wiring.sh @@ -12,12 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -set -ev +set -e # Execution example: lxc-veth-wiring.sh testing VETHTEST eth1 br-mgmt # CLI variables CONTAINER_NAME="${1}" +export CPID=$(lxc-info -Hpn ${CONTAINER_NAME}); VETH="${2}" INTERFACE="${3}" BRIDGE="${4}" @@ -29,14 +30,21 @@ PID="$(lxc-info -pHn ${CONTAINER_NAME})" # Exit 0 means no change, exit 3 is changed, any other exit is fail. EXIT_CODE=0 +function ns_cmd { + nsenter --mount=/proc/$CPID/ns/mnt \ + --net=/proc/$CPID/ns/net \ + --pid=/proc/$CPID/ns/pid \ + --uts=/proc/$CPID/ns/uts \ + --ipc=/proc/$CPID/ns/ipc -- $@ +} + if ! ip a l "${VETH}";then ip link add name "${VETH}" type veth peer name "${VETH_PEER}" - ip link set dev "${VETH}" up - EXIT=3 -else - ip link set dev "${VETH}" up + EXIT_CODE=3 fi +ip link set dev "${VETH}" up + if ip a l "${VETH_PEER}";then ip link set dev "${VETH_PEER}" up ip link set dev "${VETH_PEER}" netns "${PID}" name "${INTERFACE}" @@ -45,13 +53,10 @@ fi if ! brctl show "${BRIDGE}" | grep -q "${VETH}"; then brctl addif "${BRIDGE}" "${VETH}" - EXIT=3 + EXIT_CODE=3 fi -lxc-attach --name "${CONTAINER_NAME}" < lxc-stop --name {{ inventory_hostname }} --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log @@ -32,6 +32,7 @@ delay: 2 when: - lxc_container_allow_restarts | bool + listen: Lxc container restart # Due to https://github.com/ansible/ansible-modules-extras/issues/2691 # this uses the LXC CLI tools to ensure that we get logging. @@ -45,3 +46,62 @@ register: container_start until: container_start | success retries: 3 + listen: Lxc container restart + +- name: Flush routes + command: "ip route flush table main" + listen: Restart networkd + +- name: Restart systemd-networkd + systemd: + name: systemd-networkd + state: restarted + enabled: true + daemon_reload: true + listen: Restart networkd + +- name: Enable resolved + systemd: + name: systemd-resolved + state: started + enabled: true + daemon_reload: true + when: + - lxc_container_enable_resolved | bool + +- name: Enable hostnamed + systemd: + name: systemd-hostnamed + state: started + enabled: true + daemon_reload: true + +- name: Enable dbus + systemd: + name: dbus + state: started + enabled: true + daemon_reload: true + +- name: Reload systemd daemon + systemd: + daemon_reload: true + +# NOTE(hwoarang) openSUSE randomly fails to start the service +# with an error like the following one +# sysctl-container.service: Failed at step CGROUP spawning /sbin/sysctl: No such device +# Until this is fixed, we workaround it by simply retrying a few more times +# before giving up +# https://bugzilla.suse.com/show_bug.cgi?id=1055426 +# https://bugs.launchpad.net/openstack-ansible/+bug/1712741 +- name: Enable container sysctl service + service: + name: "sysctl-container" + state: started + enabled: yes + daemon_reload: yes + remote_user: root + register: _sysctl_service_started + until: _sysctl_service_started|success + retries: 5 + delay: 5 diff --git a/meta/main.yml b/meta/main.yml index c0a409c..014c4b2 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -38,4 +38,5 @@ galaxy_info: - python - development - openstack -dependencies: [] +dependencies: + - plugins diff --git a/releasenotes/notes/legacy-networking-convert-networkd-5b514e604df7c429.yaml b/releasenotes/notes/legacy-networking-convert-networkd-5b514e604df7c429.yaml new file mode 100644 index 0000000..d8f720d --- /dev/null +++ b/releasenotes/notes/legacy-networking-convert-networkd-5b514e604df7c429.yaml @@ -0,0 +1,9 @@ +--- +features: + - Within the lxc-container-create role the legacy networking scripts have been + converted to use systemd-networkd for ubuntu and centos. This gives us a + single, common, networking functionality to across multiple distros. + - All of the pre/post up, and pre/post down adhoc command options have been + converted to using systemd "oneshot" services. This conversion allows all + supported distros to benifit from the ability to run adhoc commands before + and after networking is available on both start-up and shut-down. diff --git a/tasks/lxc_container_config.yml b/tasks/lxc_container_config.yml index 085a454..0e8c122 100644 --- a/tasks/lxc_container_config.yml +++ b/tasks/lxc_container_config.yml @@ -13,13 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Execute container commands - lxc_container: - name: "{{ inventory_hostname }}" - container_command: | - {{ lxc_container_commands }} - delegate_to: "{{ physical_host }}" - +# CONTAINER SETUP AND CONFIG - name: Write default container config lineinfile: dest: "/var/lib/lxc/{{ inventory_hostname }}/config" @@ -27,6 +21,7 @@ backup: "true" with_items: "{{ lxc_container_default_config_list | union(lxc_container_config_list) }}" delegate_to: "{{ physical_host }}" + register: default_configuration_container notify: - Lxc container restart @@ -45,40 +40,77 @@ backup: "true" with_items: "{{ lxc_container_default_bind_mounts | union(lxc_container_bind_mounts) }}" delegate_to: "{{ physical_host }}" + register: bind_configuration_container notify: - Lxc container restart -# NOTE(cloudnull): Should a container already be up and running with a defined container interface -# the shell command will use the MAC address already set within the container as -# it's value. This allows the tasks to remain idempotent while ensuring that a -# container restart isn't required to set a static mac. -# This task is being done to allow a container to have a static mac address. -# before this task a container had a dynamic mac address which would -# change when a container was restarted. This restart process causes terrible -# issues in several network intensive systems (RabbitMQ, Neutron, etc). To -# resolve the rotating mac address issue this task is setting the mac in a hwaddr -# file and a lookup is being used in the container-interface.ini template to render -# the static hardware address as expected. -- name: Set define static mac address from an existing interface - shell: | - C_PID="$(lxc-info --name {{ inventory_hostname }} --pid | awk '/PID:/ {print $2}')" - C_ADDR="/proc/${C_PID}/root/sys/class/net/{{ item.value.interface }}/address" - HARDWARE_ADDR="/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" - HEXCHARS="0123456789abcdef" - if ! cat "${C_ADDR}" > "${HARDWARE_ADDR}"; then - echo "00:16:3e$( - for i in {1..6}; do - echo -n "${HEXCHARS:$(( $RANDOM % 16 )):1}" - done | sed -e 's/\(..\)/:\1/g' - )" > "${HARDWARE_ADDR}" - fi - args: - executable: /bin/bash - creates: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" - with_dict: "{{ container_networks | default({}) }}" +### REMOVE IN "S" +# NOTE(cloudnull): These tasks are cleaning up the old interfaces +# files. Remove this in the "S" release. +- name: Remove legacy network config for eth0 + lineinfile: + dest: "/var/lib/lxc/{{ inventory_hostname }}/config" + regexp: "^lxc.network.*" + backup: "true" + state: "absent" delegate_to: "{{ physical_host }}" - tags: - - skip_ansible_lint + +- name: Create and start the container + lxc_container: + name: "{{ inventory_hostname }}" + state: started + delegate_to: "{{ physical_host }}" + +- name: Gather container facts + setup: + gather_subset: "!all" + +- name: Drop container setup script + template: + src: "container-setup.sh.j2" + dest: "/opt/container-setup.sh" + owner: "root" + group: "root" + mode: "0755" + +- name: Drop container first run script + template: + src: "container-first-run.sh.j2" + dest: "/var/lib/lxc/{{ inventory_hostname }}/container-first-run.sh" + owner: "root" + group: "root" + mode: "0755" + delegate_to: "{{ physical_host }}" + +- name: Execute first script + command: "/var/lib/lxc/{{ inventory_hostname }}/container-first-run.sh" + args: + creates: "/var/lib/lxc/{{ inventory_hostname }}/setup.complete" + register: container_extra_commands + until: container_extra_commands | success + retries: 5 + delay: 2 + delegate_to: "{{ physical_host }}" +# CONTAINER SETUP AND CONFIG + + +# VETH AND CONNECTIVITY SETTINGS +- name: Create container mac script + template: + src: container_mac_generation.sh.j2 + dest: "/openstack/{{ inventory_hostname }}/{{ item.value.interface }}_mac_generation.sh" + mode: "0755" + delegate_to: "{{ physical_host }}" + with_dict: "{{ lxc_container_networks_combined }}" + when: + - lxc_container_fixed_mac | bool + +- name: Set define static mac address from an existing interface + command: "/openstack/{{ inventory_hostname }}/{{ item.value.interface }}_mac_generation.sh" + args: + creates: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" + delegate_to: "{{ physical_host }}" + with_dict: "{{ lxc_container_networks_combined }}" when: - lxc_container_fixed_mac | bool @@ -87,12 +119,13 @@ src: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" changed_when: false register: macs - with_dict: "{{ container_networks | default({}) }}" + with_dict: "{{ lxc_container_networks_combined }}" delegate_to: "{{ physical_host }}" when: - lxc_container_fixed_mac | bool -# NOTE(cloudnull): To dynamically set the the mac address "facts" Ansible line format is being used +# NOTE(cloudnull): To dynamically set the the mac address "facts" Ansible line +# format is being used - name: Set fixed hardware address fact set_fact: "{{item.item.value.interface }}_mac_address={{ item.content | b64decode }}" with_items: @@ -107,61 +140,8 @@ owner: "root" group: "root" mode: "0644" - with_dict: "{{ container_networks | default({}) }}" - delegate_to: "{{ physical_host }}" - -- name: Create start - lxc_container: - name: "{{ inventory_hostname }}" - state: started - delegate_to: "{{ physical_host }}" - -- name: Drop container network file (interfaces) - template: - src: "{{ lxc_container_interface }}" - dest: "{{ lxc_container_interface_target }}" - owner: "root" - group: "root" - mode: "0644" - with_dict: "{{ container_networks | default({}) }}" - -- name: Drop container network file (routes) - template: - src: "{{ lxc_container_route_interface }}" - dest: "{{ lxc_container_default_route_interfaces }}" - owner: "root" - group: "root" - mode: "0644" - when: - - lxc_container_route_interface is defined - - lxc_container_default_route_interfaces is defined - - item.value.static_routes is defined or - (item.value.gateway is defined and ansible_pkg_mgr == "zypper") - with_dict: "{{ container_networks | default({}) }}" - -- name: Drop container setup script - template: - src: "container-setup.sh.j2" - dest: "/opt/container-setup.sh" - owner: "root" - group: "root" - mode: "0755" - -- name: Run container setup script - command: /opt/container-setup.sh - register: container_setup - changed_when: false - failed_when: container_setup.rc != 0 - -# NOTE(major): the lxc.network.veth.pair line must appear *immediately* after -# the lxc.network.name congfiguration line or it will be ignored. That's why -# you'll find a "insertafter" in this YAML block. -- name: Add veth pair name to match container name - lineinfile: - dest: "/var/lib/lxc/{{ inventory_hostname }}/config" - line: "lxc.network.veth.pair = {{ lxc_container_network_veth_pair_prefix }}_eth0" - insertafter: "^lxc.network.name" - backup: "true" + with_dict: "{{ lxc_container_networks_combined }}" + register: network_config delegate_to: "{{ physical_host }}" - name: Container network includes @@ -169,35 +149,20 @@ dest: "/var/lib/lxc/{{ inventory_hostname }}/config" line: "lxc.include = /var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.ini" backup: "true" - with_dict: "{{ container_networks | default({}) }}" + with_dict: "{{ lxc_container_networks_combined }}" when: item.value.interface is defined + register: network_includes delegate_to: "{{ physical_host }}" - name: Create wiring script copy: src: "lxc-veth-wiring.sh" - dest: "/usr/local/bin/lxc-veth-wiring" + dest: "/var/lib/lxc/{{ inventory_hostname }}/lxc-veth-wiring.sh" owner: "root" group: "root" mode: "0755" delegate_to: "{{ physical_host }}" -- name: Run container veth wiring script - command: > - /usr/local/bin/lxc-veth-wiring - "{{ inventory_hostname }}" - "{{ lxc_container_network_veth_pair[-15:] }}" - "{{ item.value.interface }}" - "{{ item.value.bridge }}" - register: wiring_script - with_dict: "{{ container_networks | default({}) }}" - when: - - item.value.interface is defined - - item.value.type is not defined or item.value.type == 'veth' - failed_when: wiring_script.rc not in [3, 0] - changed_when: wiring_script.rc == 3 - delegate_to: "{{ physical_host }}" - # Adds post-down and pre-start hooks - name: Drop veth cleanup script template: @@ -209,9 +174,6 @@ delegate_to: "{{ physical_host }}" # This is being defined due to an issue with dangling veth pairs. -# TODO(someone) This should be removed once an upstream patch has -# been submitted to either the kernel or LXC to fix the veth issues. -# Container restart is not happening here because it's not needed. - name: Defines a pre and post hook script lineinfile: dest: "/var/lib/lxc/{{ inventory_hostname }}/config" @@ -222,16 +184,38 @@ - "lxc.hook.post-stop = /var/lib/lxc/{{ inventory_hostname }}/veth-cleanup.sh" delegate_to: "{{ physical_host }}" -# Flush the handlers to ensure the container and networking is online. -- meta: flush_handlers +- name: Run veth wiring + set_fact: + lxc_container_veth_wiring: true + when: + - ((not lxc_container_veth_wiring | bool) and + ((network_config | changed) and (network_includes | changed))) and + not ((default_configuration_container | changed) or + (bind_configuration_container | changed) or + (machine_id | changed)) -- name: Wait for container connectivity - wait_for_connection: - connect_timeout: "{{ lxc_container_wait_params.connect_timeout | default(omit) }}" - delay: "{{ lxc_container_wait_params.delay | default(omit) }}" - sleep: "{{ lxc_container_wait_params.sleep | default(omit) }}" - timeout: "{{ lxc_container_wait_params.timeout | default(omit) }}" +- name: Run container veth wiring script + command: >- + /var/lib/lxc/{{ inventory_hostname }}/lxc-veth-wiring.sh + "{{ inventory_hostname }}" + "{{ lxc_container_network_veth_pair[-15:] }}" + "{{ item.value.interface }}" + "{{ item.value.bridge }}" + register: wiring_script + with_dict: "{{ lxc_container_networks_combined }}" + when: + - lxc_container_veth_wiring | bool + - item.value.interface is defined + - item.value.type is not defined or item.value.type == 'veth' + failed_when: wiring_script.rc not in [3, 0] + changed_when: wiring_script.rc == 3 + delegate_to: "{{ physical_host }}" +- include: "lxc_container_network_new.yml" +# VETH AND CONNECTIVITY SETTINGS + + +# ENVIRONMENT AND HOSTNAME SETTINGS - name: Add global_environment_variables to environment file blockinfile: dest: "/etc/environment" @@ -275,6 +259,64 @@ changed_when: false remote_user: root +- name: Ensure the hostnamed override directory exists + file: + path: "/etc/systemd/system/systemd-hostnamed.service.d" + state: "directory" + +- name: Create hostnamed override + template: + src: "systemd-hostnamed.conf" + dest: "/etc/systemd/system/systemd-hostnamed.service.d/hostnamed.conf" + notify: + - Enable hostnamed + +- name: Generate machine-id + command: "systemd-machine-id-setup" + args: + creates: "/etc/machine-id" + register: machine_id + notify: + - Lxc container restart + +- name: Ensure the dbus directory exists + file: + path: "/var/lib/dbus" + state: "directory" + +- name: Create dbus machine-id + copy: + src: "/etc/machine-id" + dest: "/var/lib/dbus/machine-id" + mode: "0444" + remote_src: "yes" + remote_user: root +# ENVIRONMENT AND HOSTNAME SETTINGS + + +# POST UP/DOWN SERVICES AND KERNEL SETTINGS +- name: Create post-up-down onshot service + template: + src: "post-up-down.oneshot.j2" + dest: "/etc/systemd/system/post-up-down-{{ item.value.interface }}.service" + mode: "0644" + owner: "root" + group: "root" + with_dict: "{{ lxc_container_networks_combined }}" + notify: + - Reload systemd daemon + +- name: Create pre-up-down onshot service + template: + src: "pre-up-down.oneshot.j2" + dest: "/etc/systemd/system/post-up-down-{{ item.value.interface }}.service" + mode: "0644" + owner: "root" + group: "root" + with_dict: "{{ lxc_container_networks_combined }}" + notify: + - Reload systemd daemon + - name: Ensure sysctl can be applied template: src: "sysctl-container.init.j2" @@ -283,26 +325,12 @@ owner: "root" group: "root" remote_user: root + notify: + - Enable container sysctl service +# POST UP/DOWN SERVICES AND KERNEL SETTINGS -# NOTE(hwoarang) openSUSE randomly fails to start the service -# with an error like the following one -# sysctl-container.service: Failed at step CGROUP spawning /sbin/sysctl: No such device -# Until this is fixed, we workaround it by simply retrying a few more times -# before giving up -# https://bugzilla.suse.com/show_bug.cgi?id=1055426 -# https://bugs.launchpad.net/openstack-ansible/+bug/1712741 -- name: Enable container sysctl service - service: - name: "sysctl-container" - state: started - enabled: yes - daemon_reload: yes - remote_user: root - register: _sysctl_service_started - until: _sysctl_service_started|success - retries: 5 - delay: 5 +# SET CONTAINER FACTS - name: Allow the usage of local facts file: path: /etc/ansible/facts.d/ @@ -318,3 +346,4 @@ value: "{{ properties['lxc_container_variant'] | default(lxc_container_variant) }}" tags: - always +# SET CONTAINER FACTS diff --git a/tasks/lxc_container_create.yml b/tasks/lxc_container_create.yml index f6ed0d8..d1634cc 100644 --- a/tasks/lxc_container_create.yml +++ b/tasks/lxc_container_create.yml @@ -49,7 +49,7 @@ # this uses the LXC CLI tools to ensure that we get logging. # TODO(odyssey4me): revisit this once the bug is fixed and released - name: Start the container if it is not already running - command: > + command: >- lxc-start --daemon --name {{ inventory_hostname }} @@ -62,23 +62,3 @@ delay: 5 when: - _lxc_container_state.stdout.find('STOPPED') != -1 - -- name: Generate machine-id - command: "systemd-machine-id-setup" - args: - creates: "/etc/machine-id" - notify: - - Lxc container restart - -- name: Ensure the dbus directory exists - file: - path: "/var/lib/dbus" - state: "directory" - -- name: Create dbus machine-id - copy: - src: "/etc/machine-id" - dest: "/var/lib/dbus/machine-id" - mode: "0444" - remote_src: "yes" - remote_user: root diff --git a/tasks/lxc_container_network_new.yml b/tasks/lxc_container_network_new.yml new file mode 100644 index 0000000..a0e4d72 --- /dev/null +++ b/tasks/lxc_container_network_new.yml @@ -0,0 +1,70 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create networkd directory + file: + path: "/etc/systemd/network" + state: directory + +- name: Drop container network file (interfaces) + template: + src: "container_network.network.j2" + dest: "/etc/systemd/network/{{ item.value.interface }}.network" + owner: "root" + group: "root" + mode: "0644" + with_dict: "{{ lxc_container_networks_combined }}" + notify: + - Enable dbus + - Enable resolved + - Restart networkd + +- name: Create resolved link + file: + src: "/var/run/systemd/resolve/resolv.conf" + dest: "/etc/resolv.conf" + force: true + state: link + when: + - lxc_container_enable_resolved | bool + notify: + - Enable dbus + - Enable resolved + +### REMOVE IN "S" +# NOTE(cloudnull): These tasks are cleaning up the old interfaces +# files. Remove this in the "S" release. +- name: Remove old route network interface(s) + file: + path: "{{ lxc_container_default_route_interfaces }}" + state: absent + with_dict: "{{ lxc_container_networks_combined }}" + when: + - lxc_container_default_route_interfaces is defined + +- name: Remove old network interface(s) + file: + path: "{{ lxc_container_interface_target }}" + state: absent + with_dict: "{{ lxc_container_networks_combined }}" + when: + - lxc_container_interface_target is defined + +- name: Remove old default network interface + file: + path: "{{ lxc_container_default_interface }}" + state: absent + when: + - lxc_container_default_interface is defined diff --git a/templates/container-first-run.sh.j2 b/templates/container-first-run.sh.j2 new file mode 100644 index 0000000..a1369f2 --- /dev/null +++ b/templates/container-first-run.sh.j2 @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -ev + +export CPID="$(lxc-info -Hpn {{ inventory_hostname }})" + +function update_backup_resolvers { + if [[ -e "/proc/${CPID}/root/etc/resolv.conf.setup" ]]; then + mv /proc/${CPID}/root/etc/resolv.conf.setup /proc/${CPID}/root/etc/resolv.conf + elif [[ -e "/proc/${CPID}/root/etc/resolv.conf" ]]; then + cp /proc/${CPID}/root/etc/resolv.conf /proc/${CPID}/root/etc/resolv.conf.setup + fi +} + +# If the setup is complete return as the process is complete +if [[ -f "/var/lib/lxc/{{ inventory_hostname }}/setup.complete" ]]; then + exit 0 +fi + +# Backup the resolvers as the setup script is not executed within an NS. +update_backup_resolvers + +# Use the host resolvers +cat /etc/resolv.conf > /proc/${CPID}/root/etc/resolv.conf + +# Execute +nsenter --mount=/proc/${CPID}/ns/mnt --pid=/proc/${CPID}/ns/pid \ + --uts=/proc/${CPID}/ns/uts --ipc=/proc/${CPID}/ns/ipc \ + -- /opt/container-setup.sh + +# Restore the resolvers +update_backup_resolvers + +# Mark the setup as complete +touch /var/lib/lxc/{{ inventory_hostname }}/setup.complete diff --git a/templates/container-setup.sh.j2 b/templates/container-setup.sh.j2 index f10a64e..1fb36cf 100644 --- a/templates/container-setup.sh.j2 +++ b/templates/container-setup.sh.j2 @@ -11,3 +11,6 @@ mkdir -p "{{ item['container_directory'] }}" {% endif %} {% endfor %} + +# Run extra commands +{{ lxc_container_extra_commands }} diff --git a/templates/container_mac_generation.sh.j2 b/templates/container_mac_generation.sh.j2 new file mode 100644 index 0000000..2703371 --- /dev/null +++ b/templates/container_mac_generation.sh.j2 @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# {{ ansible_managed }} + +set -e + +# NOTE(cloudnull): Should a container already be up and running with a defined container interface +# the shell command will use the MAC address already set within the container as +# it's value. This allows the tasks to remain idempotent while ensuring that a +# container restart isn't required to set a static mac. +# This task is being done to allow a container to have a static mac address. +# before this task a container had a dynamic mac address which would +# change when a container was restarted. This restart process causes terrible +# issues in several network intensive systems (RabbitMQ, Neutron, etc). To +# resolve the rotating mac address issue this task is setting the mac in a hwaddr +# file and a lookup is being used in the container-interface.ini template to render +# the static hardware address as expected. +C_PID="$(lxc-info --name {{ inventory_hostname }} --pid | awk '/PID:/ {print $2}')" +C_ADDR="/proc/${C_PID}/root/sys/class/net/{{ item.value.interface }}/address" +HARDWARE_ADDR="/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" +HEXCHARS="0123456789abcdef" +if ! cat "${C_ADDR}" > "${HARDWARE_ADDR}"; then + echo "00:16:3e$( + for i in {1..6}; do + echo -n "${HEXCHARS:$(( $RANDOM % 16 )):1}" + done | sed -e 's/\(..\)/:\1/g' + )" > "${HARDWARE_ADDR}" +fi diff --git a/templates/container_network.network.j2 b/templates/container_network.network.j2 new file mode 100644 index 0000000..5d96a51 --- /dev/null +++ b/templates/container_network.network.j2 @@ -0,0 +1,37 @@ +[Match] +Name={{ item.value.interface }} + +{% if item.value.address is defined %} +[Address] +{% set addr_cidr = (item.value.address | string + '/' + item.value.netmask | string) | ipaddr('prefix') %} +Address={{ item.value.address }}/{{ addr_cidr }} +{% else %} +[DHCP] +UseDNS=yes +UseNTP=yes +RouteMetric=20 +{% endif %} + +{% for route in item.value.static_routes | default([]) %} +[Route] +Source={{ route['cidr'] }} +Gateway={{ route['gateway'] }} + +{% endfor %} + +[Network] +{% if item.value.address is defined %} +{% set addr_cidr = (item.value.address | string + '/' + item.value.netmask | string) | ipaddr('prefix') %} +Address={{ item.value.address }}/{{ addr_cidr }} +{% else %} +DHCP=yes +{% endif %} +{% if item.value.gateway is defined %} +Gateway={{ item.value.gateway }} +{% endif %} + +[Link] +{% if item.value.mtu is defined %} +MTUBytes={{ item.value.mtu }} +{% endif %} +ARP=yes diff --git a/templates/debian-interface.cfg.j2 b/templates/debian-interface.cfg.j2 deleted file mode 100644 index d650bb5..0000000 --- a/templates/debian-interface.cfg.j2 +++ /dev/null @@ -1,33 +0,0 @@ -# {{ ansible_managed }} - -### start generated network for [ {{ item.value.interface }} ] ### -auto {{ item.value.interface }} -{% if item.value.address is defined %} -iface {{ item.value.interface }} inet static - address {{ item.value.address }} - netmask {{ item.value.netmask }} -{% if item.value.gateway is defined %} - gateway {{ item.value.gateway }} -{% endif %} - mtu {{ item.value.mtu|default(lxc_container_default_mtu) }} -{% if item.value.static_routes is defined %} -{% for route in item.value.static_routes %} - post-up ip route add {{ route['cidr'] }} via {{ route['gateway'] }} || true -{% endfor %} -{% endif %} -{% else %} -iface {{ item.value.interface }} inet manual -{% endif %} -{% for item in item.value.preup | default([]) | union(lxc_container_default_preup) %} - pre-up {{ item }} -{% endfor %} -{% for item in item.value.postup | default([]) | union(lxc_container_default_postup) %} - post-up {{ item }} -{% endfor %} -{% for item in item.value.predown | default([]) | union(lxc_container_default_predown) %} - pre-down {{ item }} -{% endfor %} -{% for item in item.value.postdown | default([]) | union(lxc_container_default_postdown) %} - post-down {{ item }} -{% endfor %} -### end generated network for [ {{ item.value.interface }} ] ### diff --git a/templates/post-up-down.oneshot.j2 b/templates/post-up-down.oneshot.j2 new file mode 100644 index 0000000..f5d2f4b --- /dev/null +++ b/templates/post-up-down.oneshot.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Post-up Post-down oneshot networking service +After=network.target + +[Service] +Type=oneshot + +{% for command in item.value.postup | default([]) | union(lxc_container_default_postup) %} +ExecStart={{ command }} +{% endfor %} + +{% for command in item.value.postdown | default([]) | union(lxc_container_default_postdown) %} +ExecStop={{ command }} +{% endfor %} + +[Install] +WantedBy=multi-user.target diff --git a/templates/pre-up-down.oneshot.j2 b/templates/pre-up-down.oneshot.j2 new file mode 100644 index 0000000..716394b --- /dev/null +++ b/templates/pre-up-down.oneshot.j2 @@ -0,0 +1,19 @@ +[Unit] +Description=Pre-up Pre-down oneshot networking service +Before=network-pre.target +Wants=network-pre.target +After=local-fs.target + +[Service] +Type=oneshot + +{% for command in item.value.preup | default([]) | union(lxc_container_default_preup) %} +ExecStart={{ command }} +{% endfor %} + +{% for command in item.value.predown | default([]) | union(lxc_container_default_predown) %} +ExecStop={{ command }} +{% endfor %} + +[Install] +WantedBy=multi-user.target diff --git a/templates/rhel-interface.j2 b/templates/rhel-interface.j2 deleted file mode 100644 index 2e3faa2..0000000 --- a/templates/rhel-interface.j2 +++ /dev/null @@ -1,18 +0,0 @@ -# {{ ansible_managed }} - -### start generated network for [ {{ item.value.interface }} ] ### -DEVICE={{ item.value.interface }} -BOOTPROTO=none -ONBOOT=yes -NM_CONTROLLED=no -TYPE=Ethernet -{% if item.value.address is defined %} -IPADDR={{ item.value.address }} -NETMASK={{ item.value.netmask }} -{% if item.value.gateway is defined %} -GATEWAY={{ item.value.gateway }} -{% endif %} -{% endif %} -MTU={{ item.value.mtu|default(lxc_container_default_mtu) }} -DELAY=0 -### end generated network for [ {{ item.value.interface }} ] ### \ No newline at end of file diff --git a/templates/rhel-routes.j2 b/templates/rhel-routes.j2 deleted file mode 100644 index cac0b32..0000000 --- a/templates/rhel-routes.j2 +++ /dev/null @@ -1,3 +0,0 @@ -{% for route in item.value.static_routes %} - {{ route['cidr'] }} via {{ route['gateway'] }} dev {{ item.value.interface }} -{% endfor %} diff --git a/templates/suse-interface.j2 b/templates/suse-interface.j2 deleted file mode 100644 index f7c52ca..0000000 --- a/templates/suse-interface.j2 +++ /dev/null @@ -1,11 +0,0 @@ -# {{ ansible_managed }} - -### start generated network for [ {{ item.value.interface }} ] ### -STARTMODE=auto -BOOTPROTO=static -{% if item.value.address is defined %} -IPADDR={{ item.value.address }} -NETMASK={{ item.value.netmask }} -{% endif %} -MTU={{ item.value.mtu|default(lxc_container_default_mtu) }} -### end generated network for [ {{ item.value.interface }} ] ### diff --git a/templates/suse-routes.j2 b/templates/suse-routes.j2 deleted file mode 100644 index b473a9d..0000000 --- a/templates/suse-routes.j2 +++ /dev/null @@ -1,7 +0,0 @@ -{% if item.value.gateway is defined %} -default {{ item.value.gateway }} -{%- endif %} -{% for route in item.value.static_routes %} -{{ route['cidr'] }} {{ route['gateway'] }} -{% endfor %} - diff --git a/templates/systemd-hostnamed.conf b/templates/systemd-hostnamed.conf new file mode 100644 index 0000000..89c6026 --- /dev/null +++ b/templates/systemd-hostnamed.conf @@ -0,0 +1,3 @@ +[Service] +PrivateNetwork=no +PrivateDevices=no diff --git a/templates/veth-cleanup.sh.j2 b/templates/veth-cleanup.sh.j2 index bd1b32a..f374abc 100644 --- a/templates/veth-cleanup.sh.j2 +++ b/templates/veth-cleanup.sh.j2 @@ -1,15 +1,9 @@ #!/usr/bin/env bash export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -# LXC eth0 is considered special and not managed by the base container_networks -# data structure. This is being added outside of the loop for this reason. -ip link del {{ lxc_container_network_veth_pair_prefix }}_eth0 || true - # Veth cleanup for items in the container_networks data structure -{% if container_networks is defined %} -{% for key, value in container_networks.items() %} -{% if value.type is not defined or value.type == 'veth' %} +{% for key, value in lxc_container_networks_combined.items() %} +{% if value.type is not defined or value.type == 'veth' %} ip link del {{ lxc_container_network_veth_pair_prefix }}_{{ value.interface }} || true -{% endif %} +{% endif %} {% endfor %} -{% endif %} diff --git a/tests/ansible-role-requirements.yml b/tests/ansible-role-requirements.yml index ac87a1c..47352a6 100644 --- a/tests/ansible-role-requirements.yml +++ b/tests/ansible-role-requirements.yml @@ -14,3 +14,7 @@ src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts scm: git version: master +- name: plugins + src: https://git.openstack.org/openstack/openstack-ansible-plugins + scm: git + version: master diff --git a/tests/test-containers-create.yml b/tests/test-containers-create.yml new file mode 100644 index 0000000..f214e3c --- /dev/null +++ b/tests/test-containers-create.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather facts for the LXC hosts + hosts: localhost + become: true + gather_facts: true + tasks: + - include: "common/common-tasks/test-set-nodepool-vars.yml" + +- include: "common/destroy_containers.yml" + when: destroy_first | default(True) | bool + +- name: Playbook for creating containers + hosts: all_containers + become: True + gather_facts: False + any_errors_fatal: true + tasks: + - name: Create containers + include_role: + name: "lxc_container_create" + vars_files: + - common/test-vars.yml diff --git a/tests/test.yml b/tests/test.yml index 4174e2f..74a2aa0 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -19,8 +19,14 @@ # Create a btrfs backend - include: test-create-btrfs-dev.yml -# Setup the host -- include: common/test-setup-host.yml +# Prepare the user ssh keys +- include: common/test-prepare-keys.yml + +# Prepare the host +- include: common/test-prepare-host.yml + +# Create the containers +- include: test-containers-create.yml # Test container creation - include: test-containers-functional.yml diff --git a/vars/main.yml b/vars/main.yml new file mode 100644 index 0000000..4c5cade --- /dev/null +++ b/vars/main.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A set of container networks used within the LXC containers. This information +# is sourced from the assumed "container_networks" and "lxc_container_networks" +# variables and combined into a single immutable hash. +lxc_container_networks_combined: "{{ container_networks | default({}) | combine(lxc_container_networks) }}" diff --git a/vars/redhat-7.yml b/vars/redhat-7.yml index 95db2b4..4528816 100644 --- a/vars/redhat-7.yml +++ b/vars/redhat-7.yml @@ -13,17 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Note this is a used in an iterable and requires the variable -# The container interface variable is a a default object that assume the -# Ansible iterator type is `with_dict`. -lxc_container_interface: rhel-interface.j2 -lxc_container_route_interface: rhel-routes.j2 +# NOTE(cloudnull): These variables have been kept to ensure +# we're cleaning up the old interface files. +# Remote this in the "R" release. lxc_container_default_route_interfaces: "/etc/sysconfig/network-scripts/route-{{ item.value.interface }}" - -# Note this is a used in an iterable and requires the variable lxc_container_interface_target: "/etc/sysconfig/network-scripts/ifcfg-{{ item.value.interface }}" +lxc_container_default_interface: "/etc/sysconfig/network-scripts/ifcfg-eth0" lxc_container_map: distro: centos arch: amd64 release: 7 + +### REMOVE IN "S" +# The networkd commands are executed within the container but using the host +# network namespace. Remove this once systemd-networkd is part of the base +# image. +_lxc_container_extra_commands: | + which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum' + timeout 90 $RHT_PKG_MGR check-update + echo "update" > /tmp/package-transaction.txt + echo "install systemd-networkd systemd-resolved" >> /tmp/package-transaction.txt + echo "run" >> /tmp/package-transaction.txt + $RHT_PKG_MGR -y shell /tmp/package-transaction.txt + systemctl enable systemd-networkd diff --git a/vars/suse-42.yml b/vars/suse-42.yml index 75ba3c9..03158ea 100644 --- a/vars/suse-42.yml +++ b/vars/suse-42.yml @@ -24,7 +24,32 @@ lxc_container_default_route_interfaces: "/etc/sysconfig/network/ifroute-{{ item. # Note this is a used in an iterable and requires the variable lxc_container_interface_target: "/etc/sysconfig/network/ifcfg-{{ item.value.interface }}" +lxc_container_enable_resolved: false + lxc_container_map: distro: opensuse arch: amd64 release: "{{ hostvars[physical_host]['ansible_distribution_version'] }}" + +### REMOVE IN "S" +# The machine-id is not removed in the base container. Remove the machine-id +# command when the base container no longer has a stored id. +# The wicked* need to be disabled. Once this is taken care of in the base image +# this set of commands can be removed. +_lxc_container_extra_commands: | + if [[ ! -f /etc/machine-id.generated ]]; then + rm /etc/machine-id || true + rm /var/lib/dbus/machine-id || true + /usr/bin/systemd-machine-id-setup + cp /etc/machine-id /var/lib/dbus/machine-id + touch /etc/machine-id.generated + fi + for action in disable mask; do + systemctl ${action} wicked.service || true + systemctl ${action} wickedd.service || true + systemctl ${action} wickedd-auto4.service || true + systemctl ${action} wickedd-dhcp4.service || true + systemctl ${action} wickedd-dhcp6.service || true + systemctl ${action} wickedd-nanny.service || true + done + systemctl enable systemd-networkd diff --git a/vars/ubuntu-16.04.yml b/vars/ubuntu-16.04.yml index a4ba2ab..918c1e1 100644 --- a/vars/ubuntu-16.04.yml +++ b/vars/ubuntu-16.04.yml @@ -13,21 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Note this is a used in an iterable and requires the variable -# The container interface variable is a a default object that assume the -# Ansible iterator type is `with_dict`. -lxc_container_interface: debian-interface.cfg.j2 - -# Note this is a used in an iterable and requires the variable +# NOTE(cloudnull): These variables have been kept to ensure +# we're cleaning up the old interface files. +# Remote this in the "R" release. +lxc_container_default_route_interfaces: "/etc/network/interfaces.d/{{ item.value.interface }}.cfg" lxc_container_interface_target: "/etc/network/interfaces.d/{{ item.value.interface }}.cfg" +lxc_container_default_interface: "/etc/network/interfaces.d/lxc-net-bridge.cfg" lxc_container_map: distro: ubuntu arch: "{{ lxc_architecture_mapping.get( hostvars[physical_host]['ansible_architecture'] | lower ) }}" release: xenial -lxc_container_default_postup: - # needed to enable gratuitous arps on interface events - - "sysctl -w net.ipv4.conf.$IFACE.arp_notify=1" - # needed to force an interface event (setting mac to what it already is) - - "ip link set $IFACE address $(cat /sys/class/net/$IFACE/address)" +### REMOVE IN "S" +# DBUS is not guarenteeded to be part of the base image. This installs the +# requirement. Once DBUS is a built in dependency remove this. +# systemd-resolved is not setup in the base image, once we can ensure that all +# deployments have this service setup the systemd-resolved lines can be removed. +_lxc_container_extra_commands: | + apt-get update + apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes dbus + for action in disable mask; do + systemctl ${action} resolvconf.service || true + systemctl ${action} systemd-networkd-resolvconf-update.path || true + systemctl ${action} systemd-networkd-resolvconf-update.service || true + done