Change the container prep using more intelligent commands

When creating many containers on a single contended host, the execution
of many delegated tasks in parallel results in failure. This patch
consolidates the container prep and networking tasks leveraging the
lxc-rootfs pid path on the physical host instead of relying on delegation.

Change-Id: I0823e34286a0857b539a94604dbe9cdeb8a605f0
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

tasks/container_create.yml

@@ -116,6 +116,7 @@
 - name: Check container state
   command: "lxc-info -n {{ inventory_hostname }} --state"
   failed_when: false
+  changed_when: false
   delegate_to: "{{ physical_host }}"
   register: _lxc_container_state
 
@@ -159,14 +160,6 @@
   with_items: "{{ lxc_container_default_bind_mounts | union(lxc_container_bind_mounts) }}"
   delegate_to: "{{ physical_host }}"
 
-- name: Ensure container directories exist
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_command: |
-      [[ ! -d "{{ item['container_directory'] }}" ]] && mkdir -p "{{ item['container_directory'] }}"
-  with_items: "{{ lxc_container_default_bind_mounts | union(lxc_container_bind_mounts) }}"
-  delegate_to: "{{ physical_host }}"
-
 - name: Add bind mount configuration to container
   lineinfile:
     dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
@@ -179,27 +172,6 @@
   tags:
     - lxc-container-config
 
-- name: Container network interfaces
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_command: |
-      echo -e '{{ lxc_container_interface }}' | tee {{ lxc_container_interface_target }}
-  with_dict: "{{ container_networks | default({}) }}"
-  delegate_to: "{{ physical_host }}"
-  tags:
-    - lxc-container-networks
-
-- name: Container network route interfaces
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_command: |
-      echo -e '{{ lxc_container_default_route_interfaces.route_setup }}' | tee {{ lxc_container_default_route_interfaces.route_file }}
-  when: item.value.static_routes is defined
-  with_dict: "{{ container_networks | default({}) }}"
-  delegate_to: "{{ physical_host }}"
-  tags:
-    - lxc-container-networks
-
 - name: LXC host config for container networks
   template:
     src: "container-interface.ini.j2"
@@ -212,6 +184,76 @@
   tags:
     - lxc-container-networks
 
+- name: Create start
+  lxc_container:
+    name: "{{ inventory_hostname }}"
+    state: started
+  delegate_to: "{{ physical_host }}"
+  tags:
+    - lxc-container-create
+
+- name: Get LXC container PID
+  command: >
+    lxc-info -pHn {{ inventory_hostname }}
+  register: container_pid
+  changed_when: false
+  delegate_to: "{{ physical_host }}"
+  tags:
+    - lxc-container-setup
+
+- name: Drop container network file (interfaces)
+  template:
+    src: "{{ lxc_container_interface }}"
+    dest: "/proc/{{ container_pid.stdout }}/root{{ lxc_container_interface_target }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  with_dict: "{{ container_networks | default({}) }}"
+  delegate_to: "{{ physical_host }}"
+  tags:
+    - lxc-container-setup
+
+- name: Drop container network file (routes)
+  template:
+    src: "{{ lxc_container_route_interface }}"
+    dest: "/proc/{{ container_pid.stdout }}/root{{ lxc_container_default_route_interfaces }}"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+  when:
+    - lxc_container_route_interface | bool
+    - lxc_container_default_route_interfaces | bool
+    - item.value.static_routes is defined
+  with_dict: "{{ container_networks | default({}) }}"
+  delegate_to: "{{ physical_host }}"
+  tags:
+    - lxc-container-setup
+
+- name: Drop container setup script
+  template:
+    src: "container-setup.sh.j2"
+    dest: "/proc/{{ container_pid.stdout }}/root/opt/container-setup.sh"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+  with_dict: "{{ container_networks | default({}) }}"
+  delegate_to: "{{ physical_host }}"
+  tags:
+    - lxc-container-setup
+
+- name: Run container setup script
+  command: |
+    lxc-attach --name "{{ inventory_hostname }}" \
+               --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log \
+               --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }} \
+               -- /opt/container-setup.sh
+  register: container_setup
+  changed_when: false
+  failed_when: container_setup.rc != 0
+  delegate_to: "{{ physical_host }}"
+  tags:
+    - lxc-container-setup
+
 # NOTE(major): the lxc.network.veth.pair line must appear *immediately* after
 # the lxc.network.name congfiguration line or it will be ignored.  That's why
 # you'll find a "insertafter" in this YAML block.

templates/container-setup.sh.j2

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -ev
+
+# Generate the lxc container create prep commands
+{{ lxc_container_commands }}
+
+{% for item in lxc_container_default_bind_mounts | union(lxc_container_bind_mounts) %}
+{%   if item['container_directory'] is defined %}
+# Create dir "{{ item['container_directory'] }}"
+mkdir -p "{{ item['container_directory'] }}"
+
+{%   endif %}
+{% endfor %}

templates/debian-interface.cfg.j2

@@ -0,0 +1,25 @@
+# {{ ansible_managed }}
+
+### start generated network for [ {{ item.value.interface }} ] ###
+auto {{ item.value.interface }}
+{% if item.value.address is defined %}
+iface {{ item.value.interface }} inet static
+    address {{ item.value.address }}
+    netmask {{ item.value.netmask }}
+{% if item.value.gateway is defined %}
+    gateway {{ item.value.gateway }}
+{% endif %}
+    mtu {{ item.value.mtu|default(lxc_container_default_mtu) }}
+    # needed to enable gratuitous arps on interface events
+    post-up sysctl -w net.ipv4.conf.$IFACE.arp_notify=1
+    # needed to force an interface event (setting mac to what it already is)
+    post-up ip link set $IFACE address $(cat /sys/class/net/$IFACE/address)
+{% if item.value.static_routes is defined %}
+{% for route in item.value.static_routes %}
+    post-up ip route add {{ route['cidr'] }} via {{ route['gateway'] }} || true
+{% endfor %}
+{% endif %}
+{% else %}
+iface {{ item.value.interface }} inet manual
+{% endif %}
+### end generated network for [ {{ item.value.interface }} ] ###

templates/rhel-interface.j2

@@ -0,0 +1,18 @@
+# {{ ansible_managed }}
+
+### start generated network for [ {{ item.value.interface }} ] ###
+DEVICE={{ item.value.interface }}
+BOOTPROTO=none
+ONBOOT=yes
+NM_CONTROLLED=no
+TYPE=Ethernet
+{% if item.value.address is defined %}
+IPADDR={{ item.value.address }}
+NETMASK={{ item.value.netmask }}
+{% if item.value.gateway is defined %}
+GATEWAY={{ item.value.gateway }}
+{% endif %}
+{% endif %}
+MTU={{ item.value.mtu|default(lxc_container_default_mtu) }}
+DELAY=0
+### end generated network for [ {{ item.value.interface }} ] ###

templates/rhel-routes.j2

@@ -0,0 +1,3 @@
+{% for route in item.value.static_routes %}
+    {{ route['cidr'] }} via {{ route['gateway'] }} dev {{ item.value.interface }}
+{% endfor %}

tests/host_vars/container1.yml

@@ -4,4 +4,6 @@ lxc_container_config_list:
   - "lxc.mount.entry=/openstack/{{ inventory_hostname }} opt/test1 none bind 0 0"
 
 lxc_container_commands: |
-  [[ ! -d "/opt/test1" ]] && mkdir -p "/opt/test1"
+  if [[ ! -d "/opt/test1" ]]; then
+    mkdir -p "/opt/test1"
+  fi

vars/redhat-7.yml

@@ -16,30 +16,9 @@
 # Note this is a used in an iterable and requires the <item.value.interface> variable
 #   The container interface variable is a a default object that assume the
 #   Ansible iterator type is `with_dict`.
-lxc_container_interface: |
-  ### start generated network for [ {{ item.value.interface }} ] ###
-  DEVICE={{ item.value.interface }}
-  BOOTPROTO=none
-  ONBOOT=yes
-  NM_CONTROLLED=no
-  TYPE=Ethernet
-  {% if item.value.address is defined %}
-  IPADDR={{ item.value.address }}
-  NETMASK={{ item.value.netmask }}
-  {% if item.value.gateway is defined %}
-  GATEWAY={{ item.value.gateway }}
-  {% endif %}
-  {% endif %}
-  MTU={{ item.value.mtu|default(lxc_container_default_mtu) }}
-  DELAY=0
-  ### end generated network for [ {{ item.value.interface }} ] ###
-
-lxc_container_default_route_interfaces:
-  route_file: "/etc/sysconfig/network-scripts/route-{{ item.value.interface }}"
-  route_setup: |
-    {% for route in item.value.static_routes %}
-        {{ route['cidr'] }} via {{ route['gateway'] }} dev {{ item.value.interface }}
-    {% endfor %}
+lxc_container_interface: rhel-interface.j2
+lxc_container_route_interface: rhel-routes.j2
+lxc_container_default_route_interfaces: "/etc/sysconfig/network-scripts/route-{{ item.value.interface }}"
 
 # Note this is a used in an iterable and requires the <item.value.interface> variable
 lxc_container_interface_target: "/etc/sysconfig/network-scripts/ifcfg-{{ item.value.interface }}"

vars/ubuntu-14.04.yml

@@ -16,35 +16,11 @@
 # Note this is a used in an iterable and requires the <item.value.interface> variable
 #   The container interface variable is a a default object that assume the
 #   Ansible iterator type is `with_dict`.
-lxc_container_interface: |
-  ### start generated network for [ {{ item.value.interface }} ] ###
-  auto {{ item.value.interface }}
-  {% if item.value.address is defined %}
-  iface {{ item.value.interface }} inet static
-      address {{ item.value.address }}
-      netmask {{ item.value.netmask }}
-  {% if item.value.gateway is defined %}
-      gateway {{ item.value.gateway }}
-  {% endif %}
-      mtu {{ item.value.mtu|default(lxc_container_default_mtu) }}
-      # needed to enable gratuitous arps on interface events
-      post-up sysctl -w net.ipv4.conf.$IFACE.arp_notify=1
-      # needed to force an interface event (setting mac to what it already is)
-      post-up ip link set $IFACE address $(cat /sys/class/net/$IFACE/address)
-  {% if item.value.static_routes is defined %}
-  {% for route in item.value.static_routes %}
-      post-up ip route add {{ route['cidr'] }} via {{ route['gateway'] }} || true
-  {% endfor %}
-  {% endif %}
-  {% else %}
-  iface {{ item.value.interface }} inet manual
-  {% endif %}
-  ### end generated network for [ {{ item.value.interface }} ] ###
+lxc_container_interface: debian-interface.cfg.j2
 
 # Notice this is already resolved in the debian interface file
-lxc_container_default_route_interfaces:
-  route_file: "/dev/null"
-  route_setup: "This is already resolved in the debian interface file"
+lxc_container_route_interface: false
+lxc_container_default_route_interfaces: false
 
 # Note this is a used in an iterable and requires the <item.value.interface> variable
 lxc_container_interface_target: "/etc/network/interfaces.d/{{ item.value.interface }}.cfg"

vars/ubuntu-16.04.yml

@@ -16,35 +16,11 @@
 # Note this is a used in an iterable and requires the <item.value.interface> variable
 #   The container interface variable is a a default object that assume the
 #   Ansible iterator type is `with_dict`.
-lxc_container_interface: |
-  ### start generated network for [ {{ item.value.interface }} ] ###
-  auto {{ item.value.interface }}
-  {% if item.value.address is defined %}
-  iface {{ item.value.interface }} inet static
-      address {{ item.value.address }}
-      netmask {{ item.value.netmask }}
-  {% if item.value.gateway is defined %}
-      gateway {{ item.value.gateway }}
-  {% endif %}
-      mtu {{ item.value.mtu|default(lxc_container_default_mtu) }}
-      # needed to enable gratuitous arps on interface events
-      post-up sysctl -w net.ipv4.conf.$IFACE.arp_notify=1
-      # needed to force an interface event (setting mac to what it already is)
-      post-up ip link set $IFACE address $(cat /sys/class/net/$IFACE/address)
-  {% if item.value.static_routes is defined %}
-  {% for route in item.value.static_routes %}
-      post-up ip route add {{ route['cidr'] }} via {{ route['gateway'] }} || true
-  {% endfor %}
-  {% endif %}
-  {% else %}
-  iface {{ item.value.interface }} inet manual
-  {% endif %}
-  ### end generated network for [ {{ item.value.interface }} ] ###
+lxc_container_interface: debian-interface.cfg.j2
 
 # Notice this is already resolved in the debian interface file
-lxc_container_default_route_interfaces:
-  route_file: "/dev/null"
-  route_setup: "This is already resolved in the debian interface file"
+lxc_container_route_interface: false
+lxc_container_default_route_interfaces: false
 
 # Note this is a used in an iterable and requires the <item.value.interface> variable
 lxc_container_interface_target: "/etc/network/interfaces.d/{{ item.value.interface }}.cfg"