Unify container network interfaces with networkd

Unify container network interfaces using Systemd Networkd for ubuntu,
centos, and openSUSE. This change allows the role to use a single way to
configure container networks.

Care has been taken to ensure we're able to cleanly upgrade to the new
capabilities within existing environments without breaking any feature
compatibility or causing any container restarts.

It's also worth noting that all of the pre/post networking up/down
script options have been converted to systemd "oneshot" services. This
retains the ability to run adhoc scripts post network availability
while also opening up this capability, which used to be ubuntu only,
to all of our supported operating systems.

> Our usage of `lxc-attach` was removed in favor of `nsenter` to fix a
  issue where multiple `lxc-attach` commands issued to a single physical
  host could result in a hang.

> Scripts that were being generated inline have been placed into
  template files. This solves a long standing memory consumption issue
  when creating lots of containers. The old shell tasks will now be 
  executed from a generated script. While this should also help with 
  debugging, the main driver is to ensure better system stability.

> A lot of cleanup has been done throughout the task files and
  templates. In the process of updating the role to use unified
  networking a lot of duplicate tasks, scripts, and processes have
  consolidated.

> Handlers have been added for network connection wait conditions and
  to various service restarts.

> The OSA plugins have been added to this role as a dependency. We
  rely on the connection plugins throughout the stack however we were
  doing a lot of workarounds to cater to the possibility of a deployer
  running this role without them. This change simply adds the plugins
  as a known dependency which allows for a more streamlined setup.

Change-Id: I5d3ddcfa11d575648a69a04f2fb30236c2c89da3
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Kevin Carter 2017-11-27 10:56:32 -06:00 committed by Kevin Carter (cloudnull)
parent 1264f62e48
commit 815ece7454
28 changed files with 617 additions and 280 deletions

View File

@ -37,6 +37,7 @@ lxc_container_default_config_list:
- "lxc.hook.autodev=/var/lib/lxc/{{ inventory_hostname }}/autodev"
lxc_container_config_list: []
lxc_container_commands: ""
lxc_container_extra_commands: "{{ _lxc_container_extra_commands | default('echo noop') }}"
# A list of bind mounts to configure for the container, for example:
# lxc_container_bind_mounts:
@ -89,9 +90,8 @@ lxc_container_backing_store: dir
# using the directory backing.
lxc_container_vg_name: lxc
# Scripts allowing the configuration of pre/post-up/down scripts in Ubuntu
# interface files. These are merged with per-interface scripts defined in the
# container_networks dict
# Scripts allowing the configuration of pre/post-up/down scripts.
# These are run as one shot services before or after networking is available.
lxc_container_default_preup: []
lxc_container_default_postup: []
lxc_container_default_predown: []
@ -147,8 +147,21 @@ lxc_container_allow_restarts: yes
lxc_container_network_veth_pair_prefix: "{{ inventory_hostname[-8:].replace('-', '').replace('_', '') }}"
lxc_container_network_veth_pair: "{{ lxc_container_network_veth_pair_prefix }}_{{ item.value.interface }}"
# A default set of container networks used within the LXC containers.
lxc_container_networks:
lxcbr0_address:
bridge: lxcbr0
interface: eth0
type: veth
# Enable fixed mac address generation for an lxc container
lxc_container_fixed_mac: false
# Enable destroying then recreating containers
lxc_container_recreate: false
# Enable running the veth wiring script
lxc_container_veth_wiring: false
# Enable systemd-resolved
lxc_container_enable_resolved: true

View File

@ -12,12 +12,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ev
set -e
# Execution example: lxc-veth-wiring.sh testing VETHTEST eth1 br-mgmt
# CLI variables
CONTAINER_NAME="${1}"
export CPID=$(lxc-info -Hpn ${CONTAINER_NAME});
VETH="${2}"
INTERFACE="${3}"
BRIDGE="${4}"
@ -29,14 +30,21 @@ PID="$(lxc-info -pHn ${CONTAINER_NAME})"
# Exit 0 means no change, exit 3 is changed, any other exit is fail.
EXIT_CODE=0
function ns_cmd {
nsenter --mount=/proc/$CPID/ns/mnt \
--net=/proc/$CPID/ns/net \
--pid=/proc/$CPID/ns/pid \
--uts=/proc/$CPID/ns/uts \
--ipc=/proc/$CPID/ns/ipc -- $@
}
if ! ip a l "${VETH}";then
ip link add name "${VETH}" type veth peer name "${VETH_PEER}"
ip link set dev "${VETH}" up
EXIT=3
else
ip link set dev "${VETH}" up
EXIT_CODE=3
fi
ip link set dev "${VETH}" up
if ip a l "${VETH_PEER}";then
ip link set dev "${VETH_PEER}" up
ip link set dev "${VETH_PEER}" netns "${PID}" name "${INTERFACE}"
@ -45,13 +53,10 @@ fi
if ! brctl show "${BRIDGE}" | grep -q "${VETH}"; then
brctl addif "${BRIDGE}" "${VETH}"
EXIT=3
EXIT_CODE=3
fi
lxc-attach --name "${CONTAINER_NAME}" <<EOC
ip link set dev "${INTERFACE}" up
ifdown "${INTERFACE}"
ifup "${INTERFACE}"
EOC
ns_cmd ip link set dev "${INTERFACE}" down || true
ns_cmd systemctl restart systemd-networkd
exit ${EXIT}
exit ${EXIT_CODE}

View File

@ -16,7 +16,7 @@
# Due to https://github.com/ansible/ansible-modules-extras/issues/2691
# this uses the LXC CLI tools to ensure that we get logging.
# TODO(odyssey4me): revisit this once the bug is fixed and released
- name: Lxc container restart
- name: Stop Container
command: >
lxc-stop --name {{ inventory_hostname }}
--logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log
@ -32,6 +32,7 @@
delay: 2
when:
- lxc_container_allow_restarts | bool
listen: Lxc container restart
# Due to https://github.com/ansible/ansible-modules-extras/issues/2691
# this uses the LXC CLI tools to ensure that we get logging.
@ -45,3 +46,62 @@
register: container_start
until: container_start | success
retries: 3
listen: Lxc container restart
- name: Flush routes
command: "ip route flush table main"
listen: Restart networkd
- name: Restart systemd-networkd
systemd:
name: systemd-networkd
state: restarted
enabled: true
daemon_reload: true
listen: Restart networkd
- name: Enable resolved
systemd:
name: systemd-resolved
state: started
enabled: true
daemon_reload: true
when:
- lxc_container_enable_resolved | bool
- name: Enable hostnamed
systemd:
name: systemd-hostnamed
state: started
enabled: true
daemon_reload: true
- name: Enable dbus
systemd:
name: dbus
state: started
enabled: true
daemon_reload: true
- name: Reload systemd daemon
systemd:
daemon_reload: true
# NOTE(hwoarang) openSUSE randomly fails to start the service
# with an error like the following one
# sysctl-container.service: Failed at step CGROUP spawning /sbin/sysctl: No such device
# Until this is fixed, we workaround it by simply retrying a few more times
# before giving up
# https://bugzilla.suse.com/show_bug.cgi?id=1055426
# https://bugs.launchpad.net/openstack-ansible/+bug/1712741
- name: Enable container sysctl service
service:
name: "sysctl-container"
state: started
enabled: yes
daemon_reload: yes
remote_user: root
register: _sysctl_service_started
until: _sysctl_service_started|success
retries: 5
delay: 5

View File

@ -38,4 +38,5 @@ galaxy_info:
- python
- development
- openstack
dependencies: []
dependencies:
- plugins

View File

@ -0,0 +1,9 @@
---
features:
- Within the lxc-container-create role the legacy networking scripts have been
converted to use systemd-networkd for ubuntu and centos. This gives us a
single, common, networking functionality to across multiple distros.
- All of the pre/post up, and pre/post down adhoc command options have been
converted to using systemd "oneshot" services. This conversion allows all
supported distros to benifit from the ability to run adhoc commands before
and after networking is available on both start-up and shut-down.

View File

@ -13,13 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Execute container commands
lxc_container:
name: "{{ inventory_hostname }}"
container_command: |
{{ lxc_container_commands }}
delegate_to: "{{ physical_host }}"
# CONTAINER SETUP AND CONFIG
- name: Write default container config
lineinfile:
dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
@ -27,6 +21,7 @@
backup: "true"
with_items: "{{ lxc_container_default_config_list | union(lxc_container_config_list) }}"
delegate_to: "{{ physical_host }}"
register: default_configuration_container
notify:
- Lxc container restart
@ -45,40 +40,77 @@
backup: "true"
with_items: "{{ lxc_container_default_bind_mounts | union(lxc_container_bind_mounts) }}"
delegate_to: "{{ physical_host }}"
register: bind_configuration_container
notify:
- Lxc container restart
# NOTE(cloudnull): Should a container already be up and running with a defined container interface
# the shell command will use the MAC address already set within the container as
# it's value. This allows the tasks to remain idempotent while ensuring that a
# container restart isn't required to set a static mac.
# This task is being done to allow a container to have a static mac address.
# before this task a container had a dynamic mac address which would
# change when a container was restarted. This restart process causes terrible
# issues in several network intensive systems (RabbitMQ, Neutron, etc). To
# resolve the rotating mac address issue this task is setting the mac in a hwaddr
# file and a lookup is being used in the container-interface.ini template to render
# the static hardware address as expected.
- name: Set define static mac address from an existing interface
shell: |
C_PID="$(lxc-info --name {{ inventory_hostname }} --pid | awk '/PID:/ {print $2}')"
C_ADDR="/proc/${C_PID}/root/sys/class/net/{{ item.value.interface }}/address"
HARDWARE_ADDR="/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr"
HEXCHARS="0123456789abcdef"
if ! cat "${C_ADDR}" > "${HARDWARE_ADDR}"; then
echo "00:16:3e$(
for i in {1..6}; do
echo -n "${HEXCHARS:$(( $RANDOM % 16 )):1}"
done | sed -e 's/\(..\)/:\1/g'
)" > "${HARDWARE_ADDR}"
fi
args:
executable: /bin/bash
creates: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr"
with_dict: "{{ container_networks | default({}) }}"
### REMOVE IN "S"
# NOTE(cloudnull): These tasks are cleaning up the old interfaces
# files. Remove this in the "S" release.
- name: Remove legacy network config for eth0
lineinfile:
dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
regexp: "^lxc.network.*"
backup: "true"
state: "absent"
delegate_to: "{{ physical_host }}"
tags:
- skip_ansible_lint
- name: Create and start the container
lxc_container:
name: "{{ inventory_hostname }}"
state: started
delegate_to: "{{ physical_host }}"
- name: Gather container facts
setup:
gather_subset: "!all"
- name: Drop container setup script
template:
src: "container-setup.sh.j2"
dest: "/opt/container-setup.sh"
owner: "root"
group: "root"
mode: "0755"
- name: Drop container first run script
template:
src: "container-first-run.sh.j2"
dest: "/var/lib/lxc/{{ inventory_hostname }}/container-first-run.sh"
owner: "root"
group: "root"
mode: "0755"
delegate_to: "{{ physical_host }}"
- name: Execute first script
command: "/var/lib/lxc/{{ inventory_hostname }}/container-first-run.sh"
args:
creates: "/var/lib/lxc/{{ inventory_hostname }}/setup.complete"
register: container_extra_commands
until: container_extra_commands | success
retries: 5
delay: 2
delegate_to: "{{ physical_host }}"
# CONTAINER SETUP AND CONFIG
# VETH AND CONNECTIVITY SETTINGS
- name: Create container mac script
template:
src: container_mac_generation.sh.j2
dest: "/openstack/{{ inventory_hostname }}/{{ item.value.interface }}_mac_generation.sh"
mode: "0755"
delegate_to: "{{ physical_host }}"
with_dict: "{{ lxc_container_networks_combined }}"
when:
- lxc_container_fixed_mac | bool
- name: Set define static mac address from an existing interface
command: "/openstack/{{ inventory_hostname }}/{{ item.value.interface }}_mac_generation.sh"
args:
creates: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr"
delegate_to: "{{ physical_host }}"
with_dict: "{{ lxc_container_networks_combined }}"
when:
- lxc_container_fixed_mac | bool
@ -87,12 +119,13 @@
src: "/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr"
changed_when: false
register: macs
with_dict: "{{ container_networks | default({}) }}"
with_dict: "{{ lxc_container_networks_combined }}"
delegate_to: "{{ physical_host }}"
when:
- lxc_container_fixed_mac | bool
# NOTE(cloudnull): To dynamically set the the mac address "facts" Ansible line format is being used
# NOTE(cloudnull): To dynamically set the the mac address "facts" Ansible line
# format is being used
- name: Set fixed hardware address fact
set_fact: "{{item.item.value.interface }}_mac_address={{ item.content | b64decode }}"
with_items:
@ -107,61 +140,8 @@
owner: "root"
group: "root"
mode: "0644"
with_dict: "{{ container_networks | default({}) }}"
delegate_to: "{{ physical_host }}"
- name: Create start
lxc_container:
name: "{{ inventory_hostname }}"
state: started
delegate_to: "{{ physical_host }}"
- name: Drop container network file (interfaces)
template:
src: "{{ lxc_container_interface }}"
dest: "{{ lxc_container_interface_target }}"
owner: "root"
group: "root"
mode: "0644"
with_dict: "{{ container_networks | default({}) }}"
- name: Drop container network file (routes)
template:
src: "{{ lxc_container_route_interface }}"
dest: "{{ lxc_container_default_route_interfaces }}"
owner: "root"
group: "root"
mode: "0644"
when:
- lxc_container_route_interface is defined
- lxc_container_default_route_interfaces is defined
- item.value.static_routes is defined or
(item.value.gateway is defined and ansible_pkg_mgr == "zypper")
with_dict: "{{ container_networks | default({}) }}"
- name: Drop container setup script
template:
src: "container-setup.sh.j2"
dest: "/opt/container-setup.sh"
owner: "root"
group: "root"
mode: "0755"
- name: Run container setup script
command: /opt/container-setup.sh
register: container_setup
changed_when: false
failed_when: container_setup.rc != 0
# NOTE(major): the lxc.network.veth.pair line must appear *immediately* after
# the lxc.network.name congfiguration line or it will be ignored. That's why
# you'll find a "insertafter" in this YAML block.
- name: Add veth pair name to match container name
lineinfile:
dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
line: "lxc.network.veth.pair = {{ lxc_container_network_veth_pair_prefix }}_eth0"
insertafter: "^lxc.network.name"
backup: "true"
with_dict: "{{ lxc_container_networks_combined }}"
register: network_config
delegate_to: "{{ physical_host }}"
- name: Container network includes
@ -169,35 +149,20 @@
dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
line: "lxc.include = /var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.ini"
backup: "true"
with_dict: "{{ container_networks | default({}) }}"
with_dict: "{{ lxc_container_networks_combined }}"
when: item.value.interface is defined
register: network_includes
delegate_to: "{{ physical_host }}"
- name: Create wiring script
copy:
src: "lxc-veth-wiring.sh"
dest: "/usr/local/bin/lxc-veth-wiring"
dest: "/var/lib/lxc/{{ inventory_hostname }}/lxc-veth-wiring.sh"
owner: "root"
group: "root"
mode: "0755"
delegate_to: "{{ physical_host }}"
- name: Run container veth wiring script
command: >
/usr/local/bin/lxc-veth-wiring
"{{ inventory_hostname }}"
"{{ lxc_container_network_veth_pair[-15:] }}"
"{{ item.value.interface }}"
"{{ item.value.bridge }}"
register: wiring_script
with_dict: "{{ container_networks | default({}) }}"
when:
- item.value.interface is defined
- item.value.type is not defined or item.value.type == 'veth'
failed_when: wiring_script.rc not in [3, 0]
changed_when: wiring_script.rc == 3
delegate_to: "{{ physical_host }}"
# Adds post-down and pre-start hooks
- name: Drop veth cleanup script
template:
@ -209,9 +174,6 @@
delegate_to: "{{ physical_host }}"
# This is being defined due to an issue with dangling veth pairs.
# TODO(someone) This should be removed once an upstream patch has
# been submitted to either the kernel or LXC to fix the veth issues.
# Container restart is not happening here because it's not needed.
- name: Defines a pre and post hook script
lineinfile:
dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
@ -222,16 +184,38 @@
- "lxc.hook.post-stop = /var/lib/lxc/{{ inventory_hostname }}/veth-cleanup.sh"
delegate_to: "{{ physical_host }}"
# Flush the handlers to ensure the container and networking is online.
- meta: flush_handlers
- name: Run veth wiring
set_fact:
lxc_container_veth_wiring: true
when:
- ((not lxc_container_veth_wiring | bool) and
((network_config | changed) and (network_includes | changed))) and
not ((default_configuration_container | changed) or
(bind_configuration_container | changed) or
(machine_id | changed))
- name: Wait for container connectivity
wait_for_connection:
connect_timeout: "{{ lxc_container_wait_params.connect_timeout | default(omit) }}"
delay: "{{ lxc_container_wait_params.delay | default(omit) }}"
sleep: "{{ lxc_container_wait_params.sleep | default(omit) }}"
timeout: "{{ lxc_container_wait_params.timeout | default(omit) }}"
- name: Run container veth wiring script
command: >-
/var/lib/lxc/{{ inventory_hostname }}/lxc-veth-wiring.sh
"{{ inventory_hostname }}"
"{{ lxc_container_network_veth_pair[-15:] }}"
"{{ item.value.interface }}"
"{{ item.value.bridge }}"
register: wiring_script
with_dict: "{{ lxc_container_networks_combined }}"
when:
- lxc_container_veth_wiring | bool
- item.value.interface is defined
- item.value.type is not defined or item.value.type == 'veth'
failed_when: wiring_script.rc not in [3, 0]
changed_when: wiring_script.rc == 3
delegate_to: "{{ physical_host }}"
- include: "lxc_container_network_new.yml"
# VETH AND CONNECTIVITY SETTINGS
# ENVIRONMENT AND HOSTNAME SETTINGS
- name: Add global_environment_variables to environment file
blockinfile:
dest: "/etc/environment"
@ -275,6 +259,64 @@
changed_when: false
remote_user: root
- name: Ensure the hostnamed override directory exists
file:
path: "/etc/systemd/system/systemd-hostnamed.service.d"
state: "directory"
- name: Create hostnamed override
template:
src: "systemd-hostnamed.conf"
dest: "/etc/systemd/system/systemd-hostnamed.service.d/hostnamed.conf"
notify:
- Enable hostnamed
- name: Generate machine-id
command: "systemd-machine-id-setup"
args:
creates: "/etc/machine-id"
register: machine_id
notify:
- Lxc container restart
- name: Ensure the dbus directory exists
file:
path: "/var/lib/dbus"
state: "directory"
- name: Create dbus machine-id
copy:
src: "/etc/machine-id"
dest: "/var/lib/dbus/machine-id"
mode: "0444"
remote_src: "yes"
remote_user: root
# ENVIRONMENT AND HOSTNAME SETTINGS
# POST UP/DOWN SERVICES AND KERNEL SETTINGS
- name: Create post-up-down onshot service
template:
src: "post-up-down.oneshot.j2"
dest: "/etc/systemd/system/post-up-down-{{ item.value.interface }}.service"
mode: "0644"
owner: "root"
group: "root"
with_dict: "{{ lxc_container_networks_combined }}"
notify:
- Reload systemd daemon
- name: Create pre-up-down onshot service
template:
src: "pre-up-down.oneshot.j2"
dest: "/etc/systemd/system/post-up-down-{{ item.value.interface }}.service"
mode: "0644"
owner: "root"
group: "root"
with_dict: "{{ lxc_container_networks_combined }}"
notify:
- Reload systemd daemon
- name: Ensure sysctl can be applied
template:
src: "sysctl-container.init.j2"
@ -283,26 +325,12 @@
owner: "root"
group: "root"
remote_user: root
notify:
- Enable container sysctl service
# POST UP/DOWN SERVICES AND KERNEL SETTINGS
# NOTE(hwoarang) openSUSE randomly fails to start the service
# with an error like the following one
# sysctl-container.service: Failed at step CGROUP spawning /sbin/sysctl: No such device
# Until this is fixed, we workaround it by simply retrying a few more times
# before giving up
# https://bugzilla.suse.com/show_bug.cgi?id=1055426
# https://bugs.launchpad.net/openstack-ansible/+bug/1712741
- name: Enable container sysctl service
service:
name: "sysctl-container"
state: started
enabled: yes
daemon_reload: yes
remote_user: root
register: _sysctl_service_started
until: _sysctl_service_started|success
retries: 5
delay: 5
# SET CONTAINER FACTS
- name: Allow the usage of local facts
file:
path: /etc/ansible/facts.d/
@ -318,3 +346,4 @@
value: "{{ properties['lxc_container_variant'] | default(lxc_container_variant) }}"
tags:
- always
# SET CONTAINER FACTS

View File

@ -49,7 +49,7 @@
# this uses the LXC CLI tools to ensure that we get logging.
# TODO(odyssey4me): revisit this once the bug is fixed and released
- name: Start the container if it is not already running
command: >
command: >-
lxc-start
--daemon
--name {{ inventory_hostname }}
@ -62,23 +62,3 @@
delay: 5
when:
- _lxc_container_state.stdout.find('STOPPED') != -1
- name: Generate machine-id
command: "systemd-machine-id-setup"
args:
creates: "/etc/machine-id"
notify:
- Lxc container restart
- name: Ensure the dbus directory exists
file:
path: "/var/lib/dbus"
state: "directory"
- name: Create dbus machine-id
copy:
src: "/etc/machine-id"
dest: "/var/lib/dbus/machine-id"
mode: "0444"
remote_src: "yes"
remote_user: root

View File

@ -0,0 +1,70 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Create networkd directory
file:
path: "/etc/systemd/network"
state: directory
- name: Drop container network file (interfaces)
template:
src: "container_network.network.j2"
dest: "/etc/systemd/network/{{ item.value.interface }}.network"
owner: "root"
group: "root"
mode: "0644"
with_dict: "{{ lxc_container_networks_combined }}"
notify:
- Enable dbus
- Enable resolved
- Restart networkd
- name: Create resolved link
file:
src: "/var/run/systemd/resolve/resolv.conf"
dest: "/etc/resolv.conf"
force: true
state: link
when:
- lxc_container_enable_resolved | bool
notify:
- Enable dbus
- Enable resolved
### REMOVE IN "S"
# NOTE(cloudnull): These tasks are cleaning up the old interfaces
# files. Remove this in the "S" release.
- name: Remove old route network interface(s)
file:
path: "{{ lxc_container_default_route_interfaces }}"
state: absent
with_dict: "{{ lxc_container_networks_combined }}"
when:
- lxc_container_default_route_interfaces is defined
- name: Remove old network interface(s)
file:
path: "{{ lxc_container_interface_target }}"
state: absent
with_dict: "{{ lxc_container_networks_combined }}"
when:
- lxc_container_interface_target is defined
- name: Remove old default network interface
file:
path: "{{ lxc_container_default_interface }}"
state: absent
when:
- lxc_container_default_interface is defined

View File

@ -0,0 +1,34 @@
#!/usr/bin/env bash
set -ev
export CPID="$(lxc-info -Hpn {{ inventory_hostname }})"
function update_backup_resolvers {
if [[ -e "/proc/${CPID}/root/etc/resolv.conf.setup" ]]; then
mv /proc/${CPID}/root/etc/resolv.conf.setup /proc/${CPID}/root/etc/resolv.conf
elif [[ -e "/proc/${CPID}/root/etc/resolv.conf" ]]; then
cp /proc/${CPID}/root/etc/resolv.conf /proc/${CPID}/root/etc/resolv.conf.setup
fi
}
# If the setup is complete return as the process is complete
if [[ -f "/var/lib/lxc/{{ inventory_hostname }}/setup.complete" ]]; then
exit 0
fi
# Backup the resolvers as the setup script is not executed within an NS.
update_backup_resolvers
# Use the host resolvers
cat /etc/resolv.conf > /proc/${CPID}/root/etc/resolv.conf
# Execute
nsenter --mount=/proc/${CPID}/ns/mnt --pid=/proc/${CPID}/ns/pid \
--uts=/proc/${CPID}/ns/uts --ipc=/proc/${CPID}/ns/ipc \
-- /opt/container-setup.sh
# Restore the resolvers
update_backup_resolvers
# Mark the setup as complete
touch /var/lib/lxc/{{ inventory_hostname }}/setup.complete

View File

@ -11,3 +11,6 @@ mkdir -p "{{ item['container_directory'] }}"
{% endif %}
{% endfor %}
# Run extra commands
{{ lxc_container_extra_commands }}

View File

@ -0,0 +1,28 @@
#!/usr/bin/env bash
# {{ ansible_managed }}
set -e
# NOTE(cloudnull): Should a container already be up and running with a defined container interface
# the shell command will use the MAC address already set within the container as
# it's value. This allows the tasks to remain idempotent while ensuring that a
# container restart isn't required to set a static mac.
# This task is being done to allow a container to have a static mac address.
# before this task a container had a dynamic mac address which would
# change when a container was restarted. This restart process causes terrible
# issues in several network intensive systems (RabbitMQ, Neutron, etc). To
# resolve the rotating mac address issue this task is setting the mac in a hwaddr
# file and a lookup is being used in the container-interface.ini template to render
# the static hardware address as expected.
C_PID="$(lxc-info --name {{ inventory_hostname }} --pid | awk '/PID:/ {print $2}')"
C_ADDR="/proc/${C_PID}/root/sys/class/net/{{ item.value.interface }}/address"
HARDWARE_ADDR="/var/lib/lxc/{{ inventory_hostname }}/{{ item.value.interface }}.hwaddr"
HEXCHARS="0123456789abcdef"
if ! cat "${C_ADDR}" > "${HARDWARE_ADDR}"; then
echo "00:16:3e$(
for i in {1..6}; do
echo -n "${HEXCHARS:$(( $RANDOM % 16 )):1}"
done | sed -e 's/\(..\)/:\1/g'
)" > "${HARDWARE_ADDR}"
fi

View File

@ -0,0 +1,37 @@
[Match]
Name={{ item.value.interface }}
{% if item.value.address is defined %}
[Address]
{% set addr_cidr = (item.value.address | string + '/' + item.value.netmask | string) | ipaddr('prefix') %}
Address={{ item.value.address }}/{{ addr_cidr }}
{% else %}
[DHCP]
UseDNS=yes
UseNTP=yes
RouteMetric=20
{% endif %}
{% for route in item.value.static_routes | default([]) %}
[Route]
Source={{ route['cidr'] }}
Gateway={{ route['gateway'] }}
{% endfor %}
[Network]
{% if item.value.address is defined %}
{% set addr_cidr = (item.value.address | string + '/' + item.value.netmask | string) | ipaddr('prefix') %}
Address={{ item.value.address }}/{{ addr_cidr }}
{% else %}
DHCP=yes
{% endif %}
{% if item.value.gateway is defined %}
Gateway={{ item.value.gateway }}
{% endif %}
[Link]
{% if item.value.mtu is defined %}
MTUBytes={{ item.value.mtu }}
{% endif %}
ARP=yes

View File

@ -1,33 +0,0 @@
# {{ ansible_managed }}
### start generated network for [ {{ item.value.interface }} ] ###
auto {{ item.value.interface }}
{% if item.value.address is defined %}
iface {{ item.value.interface }} inet static
address {{ item.value.address }}
netmask {{ item.value.netmask }}
{% if item.value.gateway is defined %}
gateway {{ item.value.gateway }}
{% endif %}
mtu {{ item.value.mtu|default(lxc_container_default_mtu) }}
{% if item.value.static_routes is defined %}
{% for route in item.value.static_routes %}
post-up ip route add {{ route['cidr'] }} via {{ route['gateway'] }} || true
{% endfor %}
{% endif %}
{% else %}
iface {{ item.value.interface }} inet manual
{% endif %}
{% for item in item.value.preup | default([]) | union(lxc_container_default_preup) %}
pre-up {{ item }}
{% endfor %}
{% for item in item.value.postup | default([]) | union(lxc_container_default_postup) %}
post-up {{ item }}
{% endfor %}
{% for item in item.value.predown | default([]) | union(lxc_container_default_predown) %}
pre-down {{ item }}
{% endfor %}
{% for item in item.value.postdown | default([]) | union(lxc_container_default_postdown) %}
post-down {{ item }}
{% endfor %}
### end generated network for [ {{ item.value.interface }} ] ###

View File

@ -0,0 +1,17 @@
[Unit]
Description=Post-up Post-down oneshot networking service
After=network.target
[Service]
Type=oneshot
{% for command in item.value.postup | default([]) | union(lxc_container_default_postup) %}
ExecStart={{ command }}
{% endfor %}
{% for command in item.value.postdown | default([]) | union(lxc_container_default_postdown) %}
ExecStop={{ command }}
{% endfor %}
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,19 @@
[Unit]
Description=Pre-up Pre-down oneshot networking service
Before=network-pre.target
Wants=network-pre.target
After=local-fs.target
[Service]
Type=oneshot
{% for command in item.value.preup | default([]) | union(lxc_container_default_preup) %}
ExecStart={{ command }}
{% endfor %}
{% for command in item.value.predown | default([]) | union(lxc_container_default_predown) %}
ExecStop={{ command }}
{% endfor %}
[Install]
WantedBy=multi-user.target

View File

@ -1,18 +0,0 @@
# {{ ansible_managed }}
### start generated network for [ {{ item.value.interface }} ] ###
DEVICE={{ item.value.interface }}
BOOTPROTO=none
ONBOOT=yes
NM_CONTROLLED=no
TYPE=Ethernet
{% if item.value.address is defined %}
IPADDR={{ item.value.address }}
NETMASK={{ item.value.netmask }}
{% if item.value.gateway is defined %}
GATEWAY={{ item.value.gateway }}
{% endif %}
{% endif %}
MTU={{ item.value.mtu|default(lxc_container_default_mtu) }}
DELAY=0
### end generated network for [ {{ item.value.interface }} ] ###

View File

@ -1,3 +0,0 @@
{% for route in item.value.static_routes %}
{{ route['cidr'] }} via {{ route['gateway'] }} dev {{ item.value.interface }}
{% endfor %}

View File

@ -1,11 +0,0 @@
# {{ ansible_managed }}
### start generated network for [ {{ item.value.interface }} ] ###
STARTMODE=auto
BOOTPROTO=static
{% if item.value.address is defined %}
IPADDR={{ item.value.address }}
NETMASK={{ item.value.netmask }}
{% endif %}
MTU={{ item.value.mtu|default(lxc_container_default_mtu) }}
### end generated network for [ {{ item.value.interface }} ] ###

View File

@ -1,7 +0,0 @@
{% if item.value.gateway is defined %}
default {{ item.value.gateway }}
{%- endif %}
{% for route in item.value.static_routes %}
{{ route['cidr'] }} {{ route['gateway'] }}
{% endfor %}

View File

@ -0,0 +1,3 @@
[Service]
PrivateNetwork=no
PrivateDevices=no

View File

@ -1,15 +1,9 @@
#!/usr/bin/env bash
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
# LXC eth0 is considered special and not managed by the base container_networks
# data structure. This is being added outside of the loop for this reason.
ip link del {{ lxc_container_network_veth_pair_prefix }}_eth0 || true
# Veth cleanup for items in the container_networks data structure
{% if container_networks is defined %}
{% for key, value in container_networks.items() %}
{% if value.type is not defined or value.type == 'veth' %}
{% for key, value in lxc_container_networks_combined.items() %}
{% if value.type is not defined or value.type == 'veth' %}
ip link del {{ lxc_container_network_veth_pair_prefix }}_{{ value.interface }} || true
{% endif %}
{% endif %}
{% endfor %}
{% endif %}

View File

@ -14,3 +14,7 @@
src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
scm: git
version: master
- name: plugins
src: https://git.openstack.org/openstack/openstack-ansible-plugins
scm: git
version: master

View File

@ -0,0 +1,36 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Gather facts for the LXC hosts
hosts: localhost
become: true
gather_facts: true
tasks:
- include: "common/common-tasks/test-set-nodepool-vars.yml"
- include: "common/destroy_containers.yml"
when: destroy_first | default(True) | bool
- name: Playbook for creating containers
hosts: all_containers
become: True
gather_facts: False
any_errors_fatal: true
tasks:
- name: Create containers
include_role:
name: "lxc_container_create"
vars_files:
- common/test-vars.yml

View File

@ -19,8 +19,14 @@
# Create a btrfs backend
- include: test-create-btrfs-dev.yml
# Setup the host
- include: common/test-setup-host.yml
# Prepare the user ssh keys
- include: common/test-prepare-keys.yml
# Prepare the host
- include: common/test-prepare-host.yml
# Create the containers
- include: test-containers-create.yml
# Test container creation
- include: test-containers-functional.yml

19
vars/main.yml Normal file
View File

@ -0,0 +1,19 @@
---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A set of container networks used within the LXC containers. This information
# is sourced from the assumed "container_networks" and "lxc_container_networks"
# variables and combined into a single immutable hash.
lxc_container_networks_combined: "{{ container_networks | default({}) | combine(lxc_container_networks) }}"

View File

@ -13,17 +13,27 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Note this is a used in an iterable and requires the <item.value.interface> variable
# The container interface variable is a a default object that assume the
# Ansible iterator type is `with_dict`.
lxc_container_interface: rhel-interface.j2
lxc_container_route_interface: rhel-routes.j2
# NOTE(cloudnull): These variables have been kept to ensure
# we're cleaning up the old interface files.
# Remote this in the "R" release.
lxc_container_default_route_interfaces: "/etc/sysconfig/network-scripts/route-{{ item.value.interface }}"
# Note this is a used in an iterable and requires the <item.value.interface> variable
lxc_container_interface_target: "/etc/sysconfig/network-scripts/ifcfg-{{ item.value.interface }}"
lxc_container_default_interface: "/etc/sysconfig/network-scripts/ifcfg-eth0"
lxc_container_map:
distro: centos
arch: amd64
release: 7
### REMOVE IN "S"
# The networkd commands are executed within the container but using the host
# network namespace. Remove this once systemd-networkd is part of the base
# image.
_lxc_container_extra_commands: |
which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
timeout 90 $RHT_PKG_MGR check-update
echo "update" > /tmp/package-transaction.txt
echo "install systemd-networkd systemd-resolved" >> /tmp/package-transaction.txt
echo "run" >> /tmp/package-transaction.txt
$RHT_PKG_MGR -y shell /tmp/package-transaction.txt
systemctl enable systemd-networkd

View File

@ -24,7 +24,32 @@ lxc_container_default_route_interfaces: "/etc/sysconfig/network/ifroute-{{ item.
# Note this is a used in an iterable and requires the <item.value.interface> variable
lxc_container_interface_target: "/etc/sysconfig/network/ifcfg-{{ item.value.interface }}"
lxc_container_enable_resolved: false
lxc_container_map:
distro: opensuse
arch: amd64
release: "{{ hostvars[physical_host]['ansible_distribution_version'] }}"
### REMOVE IN "S"
# The machine-id is not removed in the base container. Remove the machine-id
# command when the base container no longer has a stored id.
# The wicked* need to be disabled. Once this is taken care of in the base image
# this set of commands can be removed.
_lxc_container_extra_commands: |
if [[ ! -f /etc/machine-id.generated ]]; then
rm /etc/machine-id || true
rm /var/lib/dbus/machine-id || true
/usr/bin/systemd-machine-id-setup
cp /etc/machine-id /var/lib/dbus/machine-id
touch /etc/machine-id.generated
fi
for action in disable mask; do
systemctl ${action} wicked.service || true
systemctl ${action} wickedd.service || true
systemctl ${action} wickedd-auto4.service || true
systemctl ${action} wickedd-dhcp4.service || true
systemctl ${action} wickedd-dhcp6.service || true
systemctl ${action} wickedd-nanny.service || true
done
systemctl enable systemd-networkd

View File

@ -13,21 +13,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Note this is a used in an iterable and requires the <item.value.interface> variable
# The container interface variable is a a default object that assume the
# Ansible iterator type is `with_dict`.
lxc_container_interface: debian-interface.cfg.j2
# Note this is a used in an iterable and requires the <item.value.interface> variable
# NOTE(cloudnull): These variables have been kept to ensure
# we're cleaning up the old interface files.
# Remote this in the "R" release.
lxc_container_default_route_interfaces: "/etc/network/interfaces.d/{{ item.value.interface }}.cfg"
lxc_container_interface_target: "/etc/network/interfaces.d/{{ item.value.interface }}.cfg"
lxc_container_default_interface: "/etc/network/interfaces.d/lxc-net-bridge.cfg"
lxc_container_map:
distro: ubuntu
arch: "{{ lxc_architecture_mapping.get( hostvars[physical_host]['ansible_architecture'] | lower ) }}"
release: xenial
lxc_container_default_postup:
# needed to enable gratuitous arps on interface events
- "sysctl -w net.ipv4.conf.$IFACE.arp_notify=1"
# needed to force an interface event (setting mac to what it already is)
- "ip link set $IFACE address $(cat /sys/class/net/$IFACE/address)"
### REMOVE IN "S"
# DBUS is not guarenteeded to be part of the base image. This installs the
# requirement. Once DBUS is a built in dependency remove this.
# systemd-resolved is not setup in the base image, once we can ensure that all
# deployments have this service setup the systemd-resolved lines can be removed.
_lxc_container_extra_commands: |
apt-get update
apt-get install -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" --force-yes dbus
for action in disable mask; do
systemctl ${action} resolvconf.service || true
systemctl ${action} systemd-networkd-resolvconf-update.path || true
systemctl ${action} systemd-networkd-resolvconf-update.service || true
done