diff --git a/defaults/main.yml b/defaults/main.yml index 95e8092..793bae9 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -37,6 +37,7 @@ lxc_container_default_config_list: - "lxc.hook.autodev=/var/lib/lxc/{{ inventory_hostname }}/autodev" lxc_container_config_list: [] lxc_container_commands: "" +lxc_container_extra_commands: "{{ _lxc_container_extra_commands | default('echo noop') }}" # A list of bind mounts to configure for the container, for example: # lxc_container_bind_mounts: @@ -89,9 +90,8 @@ lxc_container_backing_store: dir # using the directory backing. lxc_container_vg_name: lxc -# Scripts allowing the configuration of pre/post-up/down scripts in Ubuntu -# interface files. These are merged with per-interface scripts defined in the -# container_networks dict +# Scripts allowing the configuration of pre/post-up/down scripts. +# These are run as one shot services before or after networking is available. lxc_container_default_preup: [] lxc_container_default_postup: [] lxc_container_default_predown: [] @@ -147,8 +147,21 @@ lxc_container_allow_restarts: yes lxc_container_network_veth_pair_prefix: "{{ inventory_hostname[-8:].replace('-', '').replace('_', '') }}" lxc_container_network_veth_pair: "{{ lxc_container_network_veth_pair_prefix }}_{{ item.value.interface }}" +# A default set of container networks used within the LXC containers. +lxc_container_networks: + lxcbr0_address: + bridge: lxcbr0 + interface: eth0 + type: veth + # Enable fixed mac address generation for an lxc container lxc_container_fixed_mac: false # Enable destroying then recreating containers lxc_container_recreate: false + +# Enable running the veth wiring script +lxc_container_veth_wiring: false + +# Enable systemd-resolved +lxc_container_enable_resolved: true diff --git a/files/lxc-veth-wiring.sh b/files/lxc-veth-wiring.sh index 8e6bb17..38f4fdc 100644 --- a/files/lxc-veth-wiring.sh +++ b/files/lxc-veth-wiring.sh @@ -12,12 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -set -ev +set -e # Execution example: lxc-veth-wiring.sh testing VETHTEST eth1 br-mgmt # CLI variables CONTAINER_NAME="${1}" +export CPID=$(lxc-info -Hpn ${CONTAINER_NAME}); VETH="${2}" INTERFACE="${3}" BRIDGE="${4}" @@ -29,14 +30,21 @@ PID="$(lxc-info -pHn ${CONTAINER_NAME})" # Exit 0 means no change, exit 3 is changed, any other exit is fail. EXIT_CODE=0 +function ns_cmd { + nsenter --mount=/proc/$CPID/ns/mnt \ + --net=/proc/$CPID/ns/net \ + --pid=/proc/$CPID/ns/pid \ + --uts=/proc/$CPID/ns/uts \ + --ipc=/proc/$CPID/ns/ipc -- $@ +} + if ! ip a l "${VETH}";then ip link add name "${VETH}" type veth peer name "${VETH_PEER}" - ip link set dev "${VETH}" up - EXIT=3 -else - ip link set dev "${VETH}" up + EXIT_CODE=3 fi +ip link set dev "${VETH}" up + if ip a l "${VETH_PEER}";then ip link set dev "${VETH_PEER}" up ip link set dev "${VETH_PEER}" netns "${PID}" name "${INTERFACE}" @@ -45,13 +53,10 @@ fi if ! brctl show "${BRIDGE}" | grep -q "${VETH}"; then brctl addif "${BRIDGE}" "${VETH}" - EXIT=3 + EXIT_CODE=3 fi -lxc-attach --name "${CONTAINER_NAME}" < lxc-stop --name {{ inventory_hostname }} --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log @@ -32,6 +32,7 @@ delay: 2 when: - lxc_container_allow_restarts | bool + listen: Lxc container restart # Due to https://github.com/ansible/ansible-modules-extras/issues/2691 # this uses the LXC CLI tools to ensure that we get logging. @@ -45,3 +46,62 @@ register: container_start until: container_start | success retries: 3 + listen: Lxc container restart + +- name: Flush routes + command: "ip route flush table main" + listen: Restart networkd + +- name: Restart systemd-networkd + systemd: + name: systemd-networkd + state: restarted + enabled: true + daemon_reload: true + listen: Restart networkd + +- name: Enable resolved + systemd: + name: systemd-resolved + state: started + enabled: true + daemon_reload: true + when: + - lxc_container_enable_resolved | bool + +- name: Enable hostnamed + systemd: + name: systemd-hostnamed + state: started + enabled: true + daemon_reload: true + +- name: Enable dbus + systemd: + name: dbus + state: started + enabled: true + daemon_reload: true + +- name: Reload systemd daemon + systemd: + daemon_reload: true + +# NOTE(hwoarang) openSUSE randomly fails to start the service +# with an error like the following one +# sysctl-container.service: Failed at step CGROUP spawning /sbin/sysctl: No such device +# Until this is fixed, we workaround it by simply retrying a few more times +# before giving up +# https://bugzilla.suse.com/show_bug.cgi?id=1055426 +# https://bugs.launchpad.net/openstack-ansible/+bug/1712741 +- name: Enable container sysctl service + service: + name: "sysctl-container" + state: started + enabled: yes + daemon_reload: yes + remote_user: root + register: _sysctl_service_started + until: _sysctl_service_started|success + retries: 5 + delay: 5 diff --git a/meta/main.yml b/meta/main.yml index c0a409c..014c4b2 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -38,4 +38,5 @@ galaxy_info: - python - development - openstack -dependencies: [] +dependencies: + - plugins diff --git a/releasenotes/notes/legacy-networking-convert-networkd-5b514e604df7c429.yaml b/releasenotes/notes/legacy-networking-convert-networkd-5b514e604df7c429.yaml new file mode 100644 index 0000000..d8f720d --- /dev/null +++ b/releasenotes/notes/legacy-networking-convert-networkd-5b514e604df7c429.yaml @@ -0,0 +1,9 @@ +--- +features: + - Within the lxc-container-create role the legacy networking scripts have been + converted to use systemd-networkd for ubuntu and centos. This gives us a + single, common, networking functionality to across multiple distros. + - All of the pre/post up, and pre/post down adhoc command options have been + converted to using systemd "oneshot" services. This conversion allows all + supported distros to benifit from the ability to run adhoc commands before + and after networking is available on both start-up and shut-down. diff --git a/tasks/lxc_container_config.yml b/tasks/lxc_container_config.yml index 085a454..0e8c122 100644 --- a/tasks/lxc_container_config.yml +++ b/tasks/lxc_container_config.yml @@ -13,13 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Execute container commands - lxc_container: - name: "{{ inventory_hostname }}" - container_command: | - {{ lxc_container_commands }} - delegate_to: "{{ physical_host }}" - +# CONTAINER SETUP AND CONFIG - name: Write default container config lineinfile: dest: "/var/lib/lxc/{{ inventory_hostname }}/config" @@ -27,6 +21,7 @@ backup: "true" with_items: "{{ lxc_container_default_config_list | union(lxc_container_config_list) }}" delegate_to: "{{ physical_host }}" + register: default_configuration_container notify: - Lxc container restart @@ -45,40 +40,77 @@ backup: "true" with_items: "{{ lxc_container_default_bind_mounts | union(lxc_container_bind_mounts) }}" delegate_to: "{{ physical_host }}" + register: bind_configuration_container notify: - Lxc container restart -# NOTE(cloudnull): Should a container already be up and running with a defined container interface -# the shell command will use the MAC address already set within the container as -# it's value. This allows the tasks to remain idempotent while ensuring that a -# container restart isn't required to set a static mac. -# This task is being done to allow a container to have a static mac address. -# before this task a container had a dynamic mac address which would -# change when a container was restarted. This restart process causes terrible -# issues in several network intensive systems (RabbitMQ, Neutron, etc). To -# resolve the rotating mac address issue this task is setting the mac in a hwaddr -# file and a lookup is being used in the container-interface.ini template to render -# the static hardware address as expected. -- name: Set define static mac address from an existing interface - shell: | - C_PID="$(lxc-info --name {{ inventory_hostname }} --pid | awk '/PID:/ {print $2}')" - C_ADDR="/proc/${C_PID}/root/sys/class/net/{{ item.value.interface }}/address" - HARDWARE_ADDR="/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" - HEXCHARS="0123456789abcdef" - if ! cat "${C_ADDR}" > "${HARDWARE_ADDR}"; then - echo "00:16:3e$( - for i in {1..6}; do - echo -n "${HEXCHARS:$(( $RANDOM % 16 )):1}" - done | sed -e 's/\(..\)/:\1/g' - )" > "${HARDWARE_ADDR}" - fi - args: - executable: /bin/bash - creates: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" - with_dict: "{{ container_networks | default({}) }}" +### REMOVE IN "S" +# NOTE(cloudnull): These tasks are cleaning up the old interfaces +# files. Remove this in the "S" release. +- name: Remove legacy network config for eth0 + lineinfile: + dest: "/var/lib/lxc/{{ inventory_hostname }}/config" + regexp: "^lxc.network.*" + backup: "true" + state: "absent" delegate_to: "{{ physical_host }}" - tags: - - skip_ansible_lint + +- name: Create and start the container + lxc_container: + name: "{{ inventory_hostname }}" + state: started + delegate_to: "{{ physical_host }}" + +- name: Gather container facts + setup: + gather_subset: "!all" + +- name: Drop container setup script + template: + src: "container-setup.sh.j2" + dest: "/opt/container-setup.sh" + owner: "root" + group: "root" + mode: "0755" + +- name: Drop container first run script + template: + src: "container-first-run.sh.j2" + dest: "/var/lib/lxc/{{ inventory_hostname }}/container-first-run.sh" + owner: "root" + group: "root" + mode: "0755" + delegate_to: "{{ physical_host }}" + +- name: Execute first script + command: "/var/lib/lxc/{{ inventory_hostname }}/container-first-run.sh" + args: + creates: "/var/lib/lxc/{{ inventory_hostname }}/setup.complete" + register: container_extra_commands + until: container_extra_commands | success + retries: 5 + delay: 2 + delegate_to: "{{ physical_host }}" +# CONTAINER SETUP AND CONFIG + + +# VETH AND CONNECTIVITY SETTINGS +- name: Create container mac script + template: + src: container_mac_generation.sh.j2 + dest: "/openstack/{{ inventory_hostname }}/{{ item.value.interface }}_mac_generation.sh" + mode: "0755" + delegate_to: "{{ physical_host }}" + with_dict: "{{ lxc_container_networks_combined }}" + when: + - lxc_container_fixed_mac | bool + +- name: Set define static mac address from an existing interface + command: "/openstack/{{ inventory_hostname }}/{{ item.value.interface }}_mac_generation.sh" + args: + creates: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" + delegate_to: "{{ physical_host }}" + with_dict: "{{ lxc_container_networks_combined }}" when: - lxc_container_fixed_mac | bool @@ -87,12 +119,13 @@ src: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" changed_when: false register: macs - with_dict: "{{ container_networks | default({}) }}" + with_dict: "{{ lxc_container_networks_combined }}" delegate_to: "{{ physical_host }}" when: - lxc_container_fixed_mac | bool -# NOTE(cloudnull): To dynamically set the the mac address "facts" Ansible line format is being used +# NOTE(cloudnull): To dynamically set the the mac address "facts" Ansible line +# format is being used - name: Set fixed hardware address fact set_fact: "{{item.item.value.interface }}_mac_address={{ item.content | b64decode }}" with_items: @@ -107,61 +140,8 @@ owner: "root" group: "root" mode: "0644" - with_dict: "{{ container_networks | default({}) }}" - delegate_to: "{{ physical_host }}" - -- name: Create start - lxc_container: - name: "{{ inventory_hostname }}" - state: started - delegate_to: "{{ physical_host }}" - -- name: Drop container network file (interfaces) - template: - src: "{{ lxc_container_interface }}" - dest: "{{ lxc_container_interface_target }}" - owner: "root" - group: "root" - mode: "0644" - with_dict: "{{ container_networks | default({}) }}" - -- name: Drop container network file (routes) - template: - src: "{{ lxc_container_route_interface }}" - dest: "{{ lxc_container_default_route_interfaces }}" - owner: "root" - group: "root" - mode: "0644" - when: - - lxc_container_route_interface is defined - - lxc_container_default_route_interfaces is defined - - item.value.static_routes is defined or - (item.value.gateway is defined and ansible_pkg_mgr == "zypper") - with_dict: "{{ container_networks | default({}) }}" - -- name: Drop container setup script - template: - src: "container-setup.sh.j2" - dest: "/opt/container-setup.sh" - owner: "root" - group: "root" - mode: "0755" - -- name: Run container setup script - command: /opt/container-setup.sh - register: container_setup - changed_when: false - failed_when: container_setup.rc != 0 - -# NOTE(major): the lxc.network.veth.pair line must appear *immediately* after -# the lxc.network.name congfiguration line or it will be ignored. That's why -# you'll find a "insertafter" in this YAML block. -- name: Add veth pair name to match container name - lineinfile: - dest: "/var/lib/lxc/{{ inventory_hostname }}/config" - line: "lxc.network.veth.pair = {{ lxc_container_network_veth_pair_prefix }}_eth0" - insertafter: "^lxc.network.name" - backup: "true" + with_dict: "{{ lxc_container_networks_combined }}" + register: network_config delegate_to: "{{ physical_host }}" - name: Container network includes @@ -169,35 +149,20 @@ dest: "/var/lib/lxc/{{ inventory_hostname }}/config" line: "lxc.include = /var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.ini" backup: "true" - with_dict: "{{ container_networks | default({}) }}" + with_dict: "{{ lxc_container_networks_combined }}" when: item.value.interface is defined + register: network_includes delegate_to: "{{ physical_host }}" - name: Create wiring script copy: src: "lxc-veth-wiring.sh" - dest: "/usr/local/bin/lxc-veth-wiring" + dest: "/var/lib/lxc/{{ inventory_hostname }}/lxc-veth-wiring.sh" owner: "root" group: "root" mode: "0755" delegate_to: "{{ physical_host }}" -- name: Run container veth wiring script - command: > - /usr/local/bin/lxc-veth-wiring - "{{ inventory_hostname }}" - "{{ lxc_container_network_veth_pair[-15:] }}" - "{{ item.value.interface }}" - "{{ item.value.bridge }}" - register: wiring_script - with_dict: "{{ container_networks | default({}) }}" - when: - - item.value.interface is defined - - item.value.type is not defined or item.value.type == 'veth' - failed_when: wiring_script.rc not in [3, 0] - changed_when: wiring_script.rc == 3 - delegate_to: "{{ physical_host }}" - # Adds post-down and pre-start hooks - name: Drop veth cleanup script template: @@ -209,9 +174,6 @@ delegate_to: "{{ physical_host }}" # This is being defined due to an issue with dangling veth pairs. -# TODO(someone) This should be removed once an upstream patch has -# been submitted to either the kernel or LXC to fix the veth issues. -# Container restart is not happening here because it's not needed. - name: Defines a pre and post hook script lineinfile: dest: "/var/lib/lxc/{{ inventory_hostname }}/config" @@ -222,16 +184,38 @@ - "lxc.hook.post-stop = /var/lib/lxc/{{ inventory_hostname }}/veth-cleanup.sh" delegate_to: "{{ physical_host }}" -# Flush the handlers to ensure the container and networking is online. -- meta: flush_handlers +- name: Run veth wiring + set_fact: + lxc_container_veth_wiring: true + when: + - ((not lxc_container_veth_wiring | bool) and + ((network_config | changed) and (network_includes | changed))) and + not ((default_configuration_container | changed) or + (bind_configuration_container | changed) or + (machine_id | changed)) -- name: Wait for container connectivity - wait_for_connection: - connect_timeout: "{{ lxc_container_wait_params.connect_timeout | default(omit) }}" - delay: "{{ lxc_container_wait_params.delay | default(omit) }}" - sleep: "{{ lxc_container_wait_params.sleep | default(omit) }}" - timeout: "{{ lxc_container_wait_params.timeout | default(omit) }}" +- name: Run container veth wiring script + command: >- + /var/lib/lxc/{{ inventory_hostname }}/lxc-veth-wiring.sh + "{{ inventory_hostname }}" + "{{ lxc_container_network_veth_pair[-15:] }}" + "{{ item.value.interface }}" + "{{ item.value.bridge }}" + register: wiring_script + with_dict: "{{ lxc_container_networks_combined }}" + when: + - lxc_container_veth_wiring | bool + - item.value.interface is defined + - item.value.type is not defined or item.value.type == 'veth' + failed_when: wiring_script.rc not in [3, 0] + changed_when: wiring_script.rc == 3 + delegate_to: "{{ physical_host }}" +- include: "lxc_container_network_new.yml" +# VETH AND CONNECTIVITY SETTINGS + + +# ENVIRONMENT AND HOSTNAME SETTINGS - name: Add global_environment_variables to environment file blockinfile: dest: "/etc/environment" @@ -275,6 +259,64 @@ changed_when: false remote_user: root +- name: Ensure the hostnamed override directory exists + file: + path: "/etc/systemd/system/systemd-hostnamed.service.d" + state: "directory" + +- name: Create hostnamed override + template: + src: "systemd-hostnamed.conf" + dest: "/etc/systemd/system/systemd-hostnamed.service.d/hostnamed.conf" + notify: + - Enable hostnamed + +- name: Generate machine-id + command: "systemd-machine-id-setup" + args: + creates: "/etc/machine-id" + register: machine_id + notify: + - Lxc container restart + +- name: Ensure the dbus directory exists + file: + path: "/var/lib/dbus" + state: "directory" + +- name: Create dbus machine-id + copy: + src: "/etc/machine-id" + dest: "/var/lib/dbus/machine-id" + mode: "0444" + remote_src: "yes" + remote_user: root +# ENVIRONMENT AND HOSTNAME SETTINGS + + +# POST UP/DOWN SERVICES AND KERNEL SETTINGS +- name: Create post-up-down onshot service + template: + src: "post-up-down.oneshot.j2" + dest: "/etc/systemd/system/post-up-down-{{ item.value.interface }}.service" + mode: "0644" + owner: "root" + group: "root" + with_dict: "{{ lxc_container_networks_combined }}" + notify: + - Reload systemd daemon + +- name: Create pre-up-down onshot service + template: + src: "pre-up-down.oneshot.j2" + dest: "/etc/systemd/system/post-up-down-{{ item.value.interface }}.service" + mode: "0644" + owner: "root" + group: "root" + with_dict: "{{ lxc_container_networks_combined }}" + notify: + - Reload systemd daemon + - name: Ensure sysctl can be applied template: src: "sysctl-container.init.j2" @@ -283,26 +325,12 @@ owner: "root" group: "root" remote_user: root + notify: + - Enable container sysctl service +# POST UP/DOWN SERVICES AND KERNEL SETTINGS -# NOTE(hwoarang) openSUSE randomly fails to start the service -# with an error like the following one -# sysctl-container.service: Failed at step CGROUP spawning /sbin/sysctl: No such device -# Until this is fixed, we workaround it by simply retrying a few more times -# before giving up -# https://bugzilla.suse.com/show_bug.cgi?id=1055426 -# https://bugs.launchpad.net/openstack-ansible/+bug/1712741 -- name: Enable container sysctl service - service: - name: "sysctl-container" - state: started - enabled: yes - daemon_reload: yes - remote_user: root - register: _sysctl_service_started - until: _sysctl_service_started|success - retries: 5 - delay: 5 +# SET CONTAINER FACTS - name: Allow the usage of local facts file: path: /etc/ansible/facts.d/ @@ -318,3 +346,4 @@ value: "{{ properties['lxc_container_variant'] | default(lxc_container_variant) }}" tags: - always +# SET CONTAINER FACTS diff --git a/tasks/lxc_container_create.yml b/tasks/lxc_container_create.yml index f6ed0d8..d1634cc 100644 --- a/tasks/lxc_container_create.yml +++ b/tasks/lxc_container_create.yml @@ -49,7 +49,7 @@ # this uses the LXC CLI tools to ensure that we get logging. # TODO(odyssey4me): revisit this once the bug is fixed and released - name: Start the container if it is not already running - command: > + command: >- lxc-start --daemon --name {{ inventory_hostname }} @@ -62,23 +62,3 @@ delay: 5 when: - _lxc_container_state.stdout.find('STOPPED') != -1 - -- name: Generate machine-id - command: "systemd-machine-id-setup" - args: - creates: "/etc/machine-id" - notify: - - Lxc container restart - -- name: Ensure the dbus directory exists - file: - path: "/var/lib/dbus" - state: "directory" - -- name: Create dbus machine-id - copy: - src: "/etc/machine-id" - dest: "/var/lib/dbus/machine-id" - mode: "0444" - remote_src: "yes" - remote_user: root diff --git a/tasks/lxc_container_network_new.yml b/tasks/lxc_container_network_new.yml new file mode 100644 index 0000000..a0e4d72 --- /dev/null +++ b/tasks/lxc_container_network_new.yml @@ -0,0 +1,70 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create networkd directory + file: + path: "/etc/systemd/network" + state: directory + +- name: Drop container network file (interfaces) + template: + src: "container_network.network.j2" + dest: "/etc/systemd/network/{{ item.value.interface }}.network" + owner: "root" + group: "root" + mode: "0644" + with_dict: "{{ lxc_container_networks_combined }}" + notify: + - Enable dbus + - Enable resolved + - Restart networkd + +- name: Create resolved link + file: + src: "/var/run/systemd/resolve/resolv.conf" + dest: "/etc/resolv.conf" + force: true + state: link + when: + - lxc_container_enable_resolved | bool + notify: + - Enable dbus + - Enable resolved + +### REMOVE IN "S" +# NOTE(cloudnull): These tasks are cleaning up the old interfaces +# files. Remove this in the "S" release. +- name: Remove old route network interface(s) + file: + path: "{{ lxc_container_default_route_interfaces }}" + state: absent + with_dict: "{{ lxc_container_networks_combined }}" + when: + - lxc_container_default_route_interfaces is defined + +- name: Remove old network interface(s) + file: + path: "{{ lxc_container_interface_target }}" + state: absent + with_dict: "{{ lxc_container_networks_combined }}" + when: + - lxc_container_interface_target is defined + +- name: Remove old default network interface + file: + path: "{{ lxc_container_default_interface }}" + state: absent + when: + - lxc_container_default_interface is defined diff --git a/templates/container-first-run.sh.j2 b/templates/container-first-run.sh.j2 new file mode 100644 index 0000000..a1369f2 --- /dev/null +++ b/templates/container-first-run.sh.j2 @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -ev + +export CPID="$(lxc-info -Hpn {{ inventory_hostname }})" + +function update_backup_resolvers { + if [[ -e "/proc/${CPID}/root/etc/resolv.conf.setup" ]]; then + mv /proc/${CPID}/root/etc/resolv.conf.setup /proc/${CPID}/root/etc/resolv.conf + elif [[ -e "/proc/${CPID}/root/etc/resolv.conf" ]]; then + cp /proc/${CPID}/root/etc/resolv.conf /proc/${CPID}/root/etc/resolv.conf.setup + fi +} + +# If the setup is complete return as the process is complete +if [[ -f "/var/lib/lxc/{{ inventory_hostname }}/setup.complete" ]]; then + exit 0 +fi + +# Backup the resolvers as the setup script is not executed within an NS. +update_backup_resolvers + +# Use the host resolvers +cat /etc/resolv.conf > /proc/${CPID}/root/etc/resolv.conf + +# Execute +nsenter --mount=/proc/${CPID}/ns/mnt --pid=/proc/${CPID}/ns/pid \ + --uts=/proc/${CPID}/ns/uts --ipc=/proc/${CPID}/ns/ipc \ + -- /opt/container-setup.sh + +# Restore the resolvers +update_backup_resolvers + +# Mark the setup as complete +touch /var/lib/lxc/{{ inventory_hostname }}/setup.complete diff --git a/templates/container-setup.sh.j2 b/templates/container-setup.sh.j2 index f10a64e..1fb36cf 100644 --- a/templates/container-setup.sh.j2 +++ b/templates/container-setup.sh.j2 @@ -11,3 +11,6 @@ mkdir -p "{{ item['container_directory'] }}" {% endif %} {% endfor %} + +# Run extra commands +{{ lxc_container_extra_commands }} diff --git a/templates/container_mac_generation.sh.j2 b/templates/container_mac_generation.sh.j2 new file mode 100644 index 0000000..2703371 --- /dev/null +++ b/templates/container_mac_generation.sh.j2 @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# {{ ansible_managed }} + +set -e + +# NOTE(cloudnull): Should a container already be up and running with a defined container interface +# the shell command will use the MAC address already set within the container as +# it's value. This allows the tasks to remain idempotent while ensuring that a +# container restart isn't required to set a static mac. +# This task is being done to allow a container to have a static mac address. +# before this task a container had a dynamic mac address which would +# change when a container was restarted. This restart process causes terrible +# issues in several network intensive systems (RabbitMQ, Neutron, etc). To +# resolve the rotating mac address issue this task is setting the mac in a hwaddr +# file and a lookup is being used in the container-interface.ini template to render +# the static hardware address as expected. +C_PID="$(lxc-info --name {{ inventory_hostname }} --pid | awk '/PID:/ {print $2}')" +C_ADDR="/proc/${C_PID}/root/sys/class/net/{{ item.value.interface }}/address" +HARDWARE_ADDR="/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr" +HEXCHARS="0123456789abcdef" +if ! cat "${C_ADDR}" > "${HARDWARE_ADDR}"; then + echo "00:16:3e$( + for i in {1..6}; do + echo -n "${HEXCHARS:$(( $RANDOM % 16 )):1}" + done | sed -e 's/\(..\)/:\1/g' + )" > "${HARDWARE_ADDR}" +fi diff --git a/templates/container_network.network.j2 b/templates/container_network.network.j2 new file mode 100644 index 0000000..5d96a51 --- /dev/null +++ b/templates/container_network.network.j2 @@ -0,0 +1,37 @@ +[Match] +Name={{ item.value.interface }} + +{% if item.value.address is defined %} +[Address] +{% set addr_cidr = (item.value.address | string + '/' + item.value.netmask | string) | ipaddr('prefix') %} +Address={{ item.value.address }}/{{ addr_cidr }} +{% else %} +[DHCP] +UseDNS=yes +UseNTP=yes +RouteMetric=20 +{% endif %} + +{% for route in item.value.static_routes | default([]) %} +[Route] +Source={{ route['cidr'] }} +Gateway={{ route['gateway'] }} + +{% endfor %} + +[Network] +{% if item.value.address is defined %} +{% set addr_cidr = (item.value.address | string + '/' + item.value.netmask | string) | ipaddr('prefix') %} +Address={{ item.value.address }}/{{ addr_cidr }} +{% else %} +DHCP=yes +{% endif %} +{% if item.value.gateway is defined %} +Gateway={{ item.value.gateway }} +{% endif %} + +[Link] +{% if item.value.mtu is defined %} +MTUBytes={{ item.value.mtu }} +{% endif %} +ARP=yes diff --git a/templates/debian-interface.cfg.j2 b/templates/debian-interface.cfg.j2 deleted file mode 100644 index d650bb5..0000000 --- a/templates/debian-interface.cfg.j2 +++ /dev/null @@ -1,33 +0,0 @@ -# {{ ansible_managed }} - -### start generated network for [ {{ item.value.interface }} ] ### -auto {{ item.value.interface }} -{% if item.value.address is defined %} -iface {{ item.value.interface }} inet static - address {{ item.value.address }} - netmask {{ item.value.netmask }} -{% if item.value.gateway is defined %} - gateway {{ item.value.gateway }} -{% endif %} - mtu {{ item.value.mtu|default(lxc_container_default_mtu) }} -{% if item.value.static_routes is defined %} -{% for route in item.value.static_routes %} - post-up ip route add {{ route['cidr'] }} via {{ route['gateway'] }} || true -{% endfor %} -{% endif %} -{% else %} -iface {{ item.value.interface }} inet manual -{% endif %} -{% for item in item.value.preup | default([]) | union(lxc_container_default_preup) %} - pre-up {{ item }} -{% endfor %} -{% for item in item.value.postup | default([]) | union(lxc_container_default_postup) %} - post-up {{ item }} -{% endfor %} -{% for item in item.value.predown | default([]) | union(lxc_container_default_predown) %} - pre-down {{ item }} -{% endfor %} -{% for item in item.value.postdown | default([]) | union(lxc_container_default_postdown) %} - post-down {{ item }} -{% endfor %} -### end generated network for [ {{ item.value.interface }} ] ### diff --git a/templates/post-up-down.oneshot.j2 b/templates/post-up-down.oneshot.j2 new file mode 100644 index 0000000..f5d2f4b --- /dev/null +++ b/templates/post-up-down.oneshot.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Post-up Post-down oneshot networking service +After=network.target + +[Service] +Type=oneshot + +{% for command in item.value.postup | default([]) | union(lxc_container_default_postup) %} +ExecStart={{ command }} +{% endfor %} + +{% for command in item.value.postdown | default([]) | union(lxc_container_default_postdown) %} +ExecStop={{ command }} +{% endfor %} + +[Install] +WantedBy=multi-user.target diff --git a/templates/pre-up-down.oneshot.j2 b/templates/pre-up-down.oneshot.j2 new file mode 100644 index 0000000..716394b --- /dev/null +++ b/templates/pre-up-down.oneshot.j2 @@ -0,0 +1,19 @@ +[Unit] +Description=Pre-up Pre-down oneshot networking service +Before=network-pre.target +Wants=network-pre.target +After=local-fs.target + +[Service] +Type=oneshot + +{% for command in item.value.preup | default([]) | union(lxc_container_default_preup) %} +ExecStart={{ command }} +{% endfor %} + +{% for command in item.value.predown | default([]) | union(lxc_container_default_predown) %} +ExecStop={{ command }} +{% endfor %} + +[Install] +WantedBy=multi-user.target diff --git a/templates/rhel-interface.j2 b/templates/rhel-interface.j2 deleted file mode 100644 index 2e3faa2..0000000 --- a/templates/rhel-interface.j2 +++ /dev/null @@ -1,18 +0,0 @@ -# {{ ansible_managed }} - -### start generated network for [ {{ item.value.interface }} ] ### -DEVICE={{ item.value.interface }} -BOOTPROTO=none -ONBOOT=yes -NM_CONTROLLED=no -TYPE=Ethernet -{% if item.value.address is defined %} -IPADDR={{ item.value.address }} -NETMASK={{ item.value.netmask }} -{% if item.value.gateway is defined %} -GATEWAY={{ item.value.gateway }} -{% endif %} -{% endif %} -MTU={{ item.value.mtu|default(lxc_container_default_mtu) }} -DELAY=0 -### end generated network for [ {{ item.value.interface }} ] ### \ No newline at end of file diff --git a/templates/rhel-routes.j2 b/templates/rhel-routes.j2 deleted file mode 100644 index cac0b32..0000000 --- a/templates/rhel-routes.j2 +++ /dev/null @@ -1,3 +0,0 @@ -{% for route in item.value.static_routes %} - {{ route['cidr'] }} via {{ route['gateway'] }} dev {{ item.value.interface }} -{% endfor %} diff --git a/templates/suse-interface.j2 b/templates/suse-interface.j2 deleted file mode 100644 index f7c52ca..0000000 --- a/templates/suse-interface.j2 +++ /dev/null @@ -1,11 +0,0 @@ -# {{ ansible_managed }} - -### start generated network for [ {{ item.value.interface }} ] ### -STARTMODE=auto -BOOTPROTO=static -{% if item.value.address is defined %} -IPADDR={{ item.value.address }} -NETMASK={{ item.value.netmask }} -{% endif %} -MTU={{ item.value.mtu|default(lxc_container_default_mtu) }} -### end generated network for [ {{ item.value.interface }} ] ### diff --git a/templates/suse-routes.j2 b/templates/suse-routes.j2 deleted file mode 100644 index b473a9d..0000000 --- a/templates/suse-routes.j2 +++ /dev/null @@ -1,7 +0,0 @@ -{% if item.value.gateway is defined %} -default {{ item.value.gateway }} -{%- endif %} -{% for route in item.value.static_routes %} -{{ route['cidr'] }} {{ route['gateway'] }} -{% endfor %} - diff --git a/templates/systemd-hostnamed.conf b/templates/systemd-hostnamed.conf new file mode 100644 index 0000000..89c6026 --- /dev/null +++ b/templates/systemd-hostnamed.conf @@ -0,0 +1,3 @@ +[Service] +PrivateNetwork=no +PrivateDevices=no diff --git a/templates/veth-cleanup.sh.j2 b/templates/veth-cleanup.sh.j2 index bd1b32a..f374abc 100644 --- a/templates/veth-cleanup.sh.j2 +++ b/templates/veth-cleanup.sh.j2 @@ -1,15 +1,9 @@ #!/usr/bin/env bash export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -# LXC eth0 is considered special and not managed by the base container_networks -# data structure. This is being added outside of the loop for this reason. -ip link del {{ lxc_container_network_veth_pair_prefix }}_eth0 || true - # Veth cleanup for items in the container_networks data structure -{% if container_networks is defined %} -{% for key, value in container_networks.items() %} -{% if value.type is not defined or value.type == 'veth' %} +{% for key, value in lxc_container_networks_combined.items() %} +{% if value.type is not defined or value.type == 'veth' %} ip link del {{ lxc_container_network_veth_pair_prefix }}_{{ value.interface }} || true -{% endif %} +{% endif %} {% endfor %} -{% endif %} diff --git a/tests/ansible-role-requirements.yml b/tests/ansible-role-requirements.yml index ac87a1c..47352a6 100644 --- a/tests/ansible-role-requirements.yml +++ b/tests/ansible-role-requirements.yml @@ -14,3 +14,7 @@ src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts scm: git version: master +- name: plugins + src: https://git.openstack.org/openstack/openstack-ansible-plugins + scm: git + version: master diff --git a/tests/test-containers-create.yml b/tests/test-containers-create.yml new file mode 100644 index 0000000..f214e3c --- /dev/null +++ b/tests/test-containers-create.yml @@ -0,0 +1,36 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Gather facts for the LXC hosts + hosts: localhost + become: true + gather_facts: true + tasks: + - include: "common/common-tasks/test-set-nodepool-vars.yml" + +- include: "common/destroy_containers.yml" + when: destroy_first | default(True) | bool + +- name: Playbook for creating containers + hosts: all_containers + become: True + gather_facts: False + any_errors_fatal: true + tasks: + - name: Create containers + include_role: + name: "lxc_container_create" + vars_files: + - common/test-vars.yml diff --git a/tests/test.yml b/tests/test.yml index 4174e2f..74a2aa0 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -19,8 +19,14 @@ # Create a btrfs backend - include: test-create-btrfs-dev.yml -# Setup the host -- include: common/test-setup-host.yml +# Prepare the user ssh keys +- include: common/test-prepare-keys.yml + +# Prepare the host +- include: common/test-prepare-host.yml + +# Create the containers +- include: test-containers-create.yml # Test container creation - include: test-containers-functional.yml diff --git a/vars/main.yml b/vars/main.yml new file mode 100644 index 0000000..4c5cade --- /dev/null +++ b/vars/main.yml @@ -0,0 +1,19 @@ +--- +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A set of container networks used within the LXC containers. This information +# is sourced from the assumed "container_networks" and "lxc_container_networks" +# variables and combined into a single immutable hash. +lxc_container_networks_combined: "{{ container_networks | default({}) | combine(lxc_container_networks) }}" diff --git a/vars/redhat-7.yml b/vars/redhat-7.yml index 95db2b4..4528816 100644 --- a/vars/redhat-7.yml +++ b/vars/redhat-7.yml @@ -13,17 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Note this is a used in an iterable and requires the variable -# The container interface variable is a a default object that assume the -# Ansible iterator type is `with_dict`. -lxc_container_interface: rhel-interface.j2 -lxc_container_route_interface: rhel-routes.j2 +# NOTE(cloudnull): These variables have been kept to ensure +# we're cleaning up the old interface files. +# Remote this in the "R" release. lxc_container_default_route_interfaces: "/etc/sysconfig/network-scripts/route-{{ item.value.interface }}" - -# Note this is a used in an iterable and requires the variable lxc_container_interface_target: "/etc/sysconfig/network-scripts/ifcfg-{{ item.value.interface }}" +lxc_container_default_interface: "/etc/sysconfig/network-scripts/ifcfg-eth0" lxc_container_map: distro: centos arch: amd64 release: 7 + +### REMOVE IN "S" +# The networkd commands are executed within the container but using the host +# network namespace. Remove this once systemd-networkd is part of the base +# image. +_lxc_container_extra_commands: | + which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum' + timeout 90 $RHT_PKG_MGR check-update + echo "update" > /tmp/package-transaction.txt + echo "install systemd-networkd systemd-resolved" >> /tmp/package-transaction.txt + echo "run" >> /tmp/package-transaction.txt + $RHT_PKG_MGR -y shell /tmp/package-transaction.txt + systemctl enable systemd-networkd diff --git a/vars/suse-42.yml b/vars/suse-42.yml index 75ba3c9..03158ea 100644 --- a/vars/suse-42.yml +++ b/vars/suse-42.yml @@ -24,7 +24,32 @@ lxc_container_default_route_interfaces: "/etc/sysconfig/network/ifroute-{{ item. # Note this is a used in an iterable and requires the variable lxc_container_interface_target: "/etc/sysconfig/network/ifcfg-{{ item.value.interface }}" +lxc_container_enable_resolved: false + lxc_container_map: distro: opensuse arch: amd64 release: "{{ hostvars[physical_host]['ansible_distribution_version'] }}" + +### REMOVE IN "S" +# The machine-id is not removed in the base container. Remove the machine-id +# command when the base container no longer has a stored id. +# The wicked* need to be disabled. Once this is taken care of in the base image +# this set of commands can be removed. +_lxc_container_extra_commands: | + if [[ ! -f /etc/machine-id.generated ]]; then + rm /etc/machine-id || true + rm /var/lib/dbus/machine-id || true + /usr/bin/systemd-machine-id-setup + cp /etc/machine-id /var/lib/dbus/machine-id + touch /etc/machine-id.generated + fi + for action in disable mask; do + systemctl ${action} wicked.service || true + systemctl ${action} wickedd.service || true + systemctl ${action} wickedd-auto4.service || true + systemctl ${action} wickedd-dhcp4.service || true + systemctl ${action} wickedd-dhcp6.service || true + systemctl ${action} wickedd-nanny.service || true + done + systemctl enable systemd-networkd diff --git a/vars/ubuntu-16.04.yml b/vars/ubuntu-16.04.yml index a4ba2ab..918c1e1 100644 --- a/vars/ubuntu-16.04.yml +++ b/vars/ubuntu-16.04.yml @@ -13,21 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Note this is a used in an iterable and requires the variable -# The container interface variable is a a default object that assume the -# Ansible iterator type is `with_dict`. -lxc_container_interface: debian-interface.cfg.j2 - -# Note this is a used in an iterable and requires the variable +# NOTE(cloudnull): These variables have been kept to ensure +# we're cleaning up the old interface files. +# Remote this in the "R" release. +lxc_container_default_route_interfaces: "/etc/network/interfaces.d/{{ item.value.interface }}.cfg" lxc_container_interface_target: "/etc/network/interfaces.d/{{ item.value.interface }}.cfg" +lxc_container_default_interface: "/etc/network/interfaces.d/lxc-net-bridge.cfg" lxc_container_map: distro: ubuntu arch: "{{ lxc_architecture_mapping.get( hostvars[physical_host]['ansible_architecture'] | lower ) }}" release: xenial -lxc_container_default_postup: - # needed to enable gratuitous arps on interface events - - "sysctl -w net.ipv4.conf.$IFACE.arp_notify=1" - # needed to force an interface event (setting mac to what it already is) - - "ip link set $IFACE address $(cat /sys/class/net/$IFACE/address)" +### REMOVE IN "S" +# DBUS is not guarenteeded to be part of the base image. This installs the +# requirement. Once DBUS is a built in dependency remove this. +# systemd-resolved is not setup in the base image, once we can ensure that all +# deployments have this service setup the systemd-resolved lines can be removed. +_lxc_container_extra_commands: | + apt-get update + apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes dbus + for action in disable mask; do + systemctl ${action} resolvconf.service || true + systemctl ${action} systemd-networkd-resolvconf-update.path || true + systemctl ${action} systemd-networkd-resolvconf-update.service || true + done