Download LXC image with async via aria2

This patch adds an async task to download the LXC image using aria2
with retries and read timeouts.

Closes-Bug: 1709329
Change-Id: Ib9ec6195dcb7e0e4b18b8526f030e6738f9953e8
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

defaults/main.yml

@@ -13,10 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Environment to use when building LXC caches
-# You can set http_proxy environment variables here for example
-# if you would like to proxy all image downloads
-lxc_cache_environment: {}
+# Validate certificates when downloading LXC templates
+lxc_hosts_validate_certs: yes
 
 # Set the package install state for distribution and pip packages
 # Options are 'present' and 'latest'
@@ -126,21 +124,25 @@ lxc_cache_prep_dns:
 lxc_cache_prep_pre_commands: '## pre command skipped ##'
 lxc_cache_prep_post_commands: '## post command skipped ##'
 
+# Set the servers to download LXC images from
+# NOTE(mhayden): The main images.linuxcontainers.org site will redirect
+# requests to (us|uk).images.linuxcontainers.org upon the first request. We
+# add the mirrors here to get around some HTTP 400 errors and allow aria2 to
+# download from both mirrors at the same time.
+lxc_image_cache_server_mirrors:
+  - https://us.images.linuxcontainers.org
+  - https://uk.images.linuxcontainers.org
+
 # The DNS name of the LXD server to source the base container cache from
-lxc_image_cache_server: images.linuxcontainers.org
-
-# The protocol to use to fetch from lxc_image_cache_server
-lxc_image_cache_server_proto: http
-
-# Path to images on server
-lxc_image_cache_server_path: "{{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}/images/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}/"
+# NOTE(cloudnull): This var should be removed in R.
+lxc_image_cache_server: "{{ lxc_image_cache_server_mirrors[0].strip('http(?s)://') }}"
 
 # Local path to cached image
 lxc_image_cache_path: "/var/lib/machines/{{ lxc_container_base_name }}"
 
 # Mode to pull image. This is used to pull the image from a remote source.
-#  Valid options are [pull-tar, pull-raw]
-lxc_image_cache_pull_mode: pull-tar
+#  Valid options are [import-tar, import-raw]
+lxc_image_cache_pull_mode: import-tar
 
 # Set this option to true to pull a new cached image.
 lxc_image_cache_refresh: false

releasenotes/notes/lxc_cache_environment-f14701a7f8f4b8ca.yaml

@@ -1,5 +0,0 @@
----
-features:
-  - The variable ``lxc_cache_environment`` has been added. This dictionary
-    can be overridden by deployers to set HTTP proxy environment variables that
-    will be applied to all lxc container download tasks.

releasenotes/notes/lxc_image_cache_server-f14701a7f8f4b8ca.yaml

@@ -0,0 +1,14 @@
+---
+features:
+  - The variable ``lxc_image_cache_server_mirrors`` has been added to
+    the "lxc_hosts" role. This is a list type variable and gives
+    deployers the ability to specify multiple lxc-image mirrors at the
+    same time.
+
+deprecations:
+  - The variable ``lxc_image_cache_server`` has been deprecated in the
+    "lxc_hosts" role. By default this value will pull the first item
+    out of ``lxc_image_cache_server_mirrors`` list which is only done
+    for compatibility (legacy) purposes. The default string type
+    variable, ``lxc_image_cache_server``, will be removed from the
+    "lxc_hosts" role in the in "R" release.

tasks/lxc_cache.yml

@@ -13,14 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Set LXC cache path fact
-  set_fact:
-    cache_path_fact: "{{ lxc_container_cache_path }}/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}"
-    cache_index_item: "{{ lxc_cache_map.distro }};{{ lxc_cache_map.release }};{{ lxc_cache_map.arch }};{{ lxc_cache_default_variant }}"
-    cache_time: "{{ ansible_date_time.epoch }}"
-  tags:
-    - always
-
 - name : Check cached image status
   command: "machinectl image-status {{ lxc_container_base_name }}"
   register: cache_check

tasks/lxc_cache_create.yml

@@ -37,13 +37,23 @@
   tags:
     - skip_ansible_lint
 
-- name: Update LXC image meta data
-  get_url:
-    url: "{{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}/{{ item.split(';')[-1] }}/meta.tar.xz"
-    dest: "/tmp/meta.tar.xz"
-  with_items: "{{ lxc_images }}"
-  retries: 5
-  delay: 2
+- name: Fetch LXC image-metadata
+  shell: >
+    aria2c
+    --max-connection-per-server=4
+    --continue
+    --allow-overwrite=true
+    --dir=/tmp
+    --out=meta.tar.xz
+    --check-certificate={{ (lxc_hosts_validate_certs | bool) | lower }}
+    {% for server in lxc_image_cache_server_mirrors %}{{ server }}/{{ item.split(';')[-1] }}/meta.tar.xz {% endfor %}
+  args:
+    warn: no
+  with_items:
+    - "{{ lxc_images }}"
+  tags:
+    - skip_ansible_lint
+    - always
 
 - name: Place container metadata
   unarchive:
@@ -51,6 +61,11 @@
     dest: "{{ cache_path_fact }}"
     remote_src: True
 
+- name: Remove metadata archive
+  file:
+    path: "/tmp/meta.tar.xz"
+    state: "absent"
+
 - name: Set cache expiry
   shell: "date -d @{{ (cache_time | int) + 31536000 }} > {{ cache_path_fact }}/expiry"
   tags:
@@ -72,7 +87,6 @@
   retries: 3
   delay: 10
   until: cache_download|success
-  environment: "{{ lxc_cache_environment }}"
   when:
     - lxc_container_backing_store is defined
     - lxc_container_backing_store == 'overlayfs'
@@ -88,7 +102,6 @@
   retries: 3
   delay: 10
   until: cache_download|success
-  environment: "{{ lxc_cache_environment }}"
   when:
     - lxc_container_backing_store is defined
     - lxc_container_backing_store == 'lvm'

tasks/lxc_cache_preparation.yml

@@ -18,25 +18,6 @@
   changed_when: false
   register: systemd_version
 
-- name: Retrieve Image Index
-  uri:
-    url: "{{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}/meta/1.0/index-system"
-    return_content: yes
-  register: image_index
-  retries: 5
-  delay: 2
-
-- name: Set image index fact
-  set_fact:
-    lxc_images: >
-      {%- set images = [] %}
-      {%- for image in image_index.content.splitlines() %}
-      {%- if image | match("^" + cache_index_item) %}
-      {%- set _ = images.append(image) %}
-      {%- endif %}
-      {%- endfor %}
-      {{- images -}}
-
 - block:
     - name: Create machined proxy override unit directories
       file:
@@ -59,10 +40,15 @@
       with_items:
         - systemd-machined.service.d
         - systemd-importd.service.d
-  when: lxc_cache_environment.keys() | length > 0
+  when: (deployment_environment_variables | default({})).keys() | length > 0
 
 - include: "lxc_cache_preparation_systemd_{{ (systemd_version.stdout_lines[0].split()[-1] | int > 219) | ternary('new', 'old') }}.yml"
 
+- name: Remove rootfs archive
+  file:
+    path: "/tmp/rootfs.tar.xz"
+    state: "absent"
+
 - name: Generate apt keys from LXC host for the container cache
   shell: apt-key exportall > /root/repo.keys
   changed_when: False

tasks/lxc_cache_preparation_systemd_new.yml

@@ -25,18 +25,27 @@
 
 # NOTE(cloudnull): When modern SystemD is running everywhere this can be
 #  collapsed back into the base preparation task file.
-- name : Set volume size
+- name: Set volume size
   shell: machinectl set-limit {{ lxc_host_machine_volume_size }}G
   changed_when: false
   args:
     executable: /bin/bash
 
+- name: Ensure image has been pre-staged
+  async_status:
+    jid: "{{ item.ansible_job_id }}"
+  register: job_result
+  until: job_result.finished
+  retries: 30
+  with_items:
+    - "{{ prestage_image.results }}"
+
 - name: Retrieve base image
   command: >-
     machinectl
     --verify=no
     {{ lxc_image_cache_pull_mode }}
-    {{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}{{ item.split(';')[-1] }}rootfs.tar.xz
+    /tmp/rootfs.tar.xz
     {{ lxc_container_base_name }}
   register: pull_image
   until: pull_image | success

tasks/lxc_cache_preparation_systemd_old.yml

@@ -115,7 +115,7 @@
 # Because of this post and it's related bug(s) this is adding the container
 #  volumes the old way. The new way would simply be calling `machinectl`.
 #  * https://www.mail-archive.com/systemd-devel@lists.freedesktop.org/msg28255.html
-- name : Remove old image cache
+- name: Remove old image cache
   command: "btrfs subvolume delete /var/lib/machines/{{ lxc_container_base_name }}"
   register: cache_refresh_del
   changed_when: cache_refresh_del.rc == 0
@@ -123,7 +123,7 @@
   when:
     - lxc_image_cache_refresh | bool
 
-- name : Add image cache
+- name: Add image cache
   command: "btrfs subvolume create /var/lib/machines/{{ lxc_container_base_name }}"
   register: cache_refresh_add
   changed_when: cache_refresh_add.rc == 0
@@ -131,23 +131,17 @@
   when:
     - lxc_image_cache_refresh | bool
 
-- name: Retrieve base image
-  get_url:
-    url: "{{ lxc_image_cache_server_proto }}://{{ lxc_image_cache_server }}{{ item.split(';')[-1] }}rootfs.tar.xz"
-    dest: "/tmp/rootfs.tar.xz"
-  register: pull_image
-  until: pull_image | success
-  retries: 5
-  delay: 2
-  with_items: "{{ lxc_images }}"
+- name: Ensure image has been pre-staged
+  async_status:
+    jid: "{{ item.ansible_job_id }}"
+  register: job_result
+  until: job_result.finished
+  retries: 30
+  with_items:
+    - "{{ prestage_image.results }}"
 
 - name: Place container rootfs
   unarchive:
     src: "/tmp/rootfs.tar.xz"
     dest: "/var/lib/machines/{{ lxc_container_base_name }}"
     remote_src: True
-
-- name: Remove rootfs archive
-  file:
-    path: "/tmp/rootfs.tar.xz"
-    state: "absent"

tasks/lxc_cache_prestage.yml

@@ -0,0 +1,78 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Set LXC cache path fact
+  set_fact:
+    cache_path_fact: "{{ lxc_container_cache_path }}/{{ lxc_cache_map.distro }}/{{ lxc_cache_map.release }}/{{ lxc_cache_map.arch }}/{{ lxc_cache_default_variant }}"
+    cache_index_item: "{{ lxc_cache_map.distro }};{{ lxc_cache_map.release }};{{ lxc_cache_map.arch }};{{ lxc_cache_default_variant }}"
+    cache_time: "{{ ansible_date_time.epoch }}"
+  tags:
+    - always
+
+- name: Fetch LXC image-index
+  shell: >
+    aria2c
+    --max-connection-per-server=4
+    --continue
+    --allow-overwrite=true
+    --dir=/tmp
+    --out=index-system
+    --check-certificate={{ (lxc_hosts_validate_certs | bool) | lower }}
+    {% for server in lxc_image_cache_server_mirrors %}{{ server }}/meta/1.0/index-system {% endfor %}
+  args:
+    warn: no
+  tags:
+    - skip_ansible_lint
+    - always
+
+- name: Retrieve the expiry object
+  slurp:
+    src: "/tmp/index-system"
+  changed_when: false
+  register: image_index
+  tags:
+    - always
+
+- name: Set image index fact
+  set_fact:
+    lxc_images: >
+      {%- set images = [] %}
+      {%- for image in (image_index.content | b64decode).splitlines() %}
+      {%- if image | match("^" + cache_index_item) %}
+      {%- set _ = images.append(image) %}
+      {%- endif %}
+      {%- endfor %}
+      {{- images -}}
+
+- name: Pre-stage the LXC image on the system
+  shell: >
+    aria2c
+    --max-connection-per-server=4
+    --continue
+    --allow-overwrite=true
+    --dir=/tmp
+    --out=rootfs.tar.xz
+    --check-certificate={{ (lxc_hosts_validate_certs | bool) | lower }}
+    {% for server in lxc_image_cache_server_mirrors %}{{ server }}{{ item.split(';')[-1] }}rootfs.tar.xz {% endfor %}
+  args:
+    warn: no
+  register: prestage_image
+  async: 300
+  poll: 0
+  with_items:
+    - "{{ lxc_images }}"
+  tags:
+    - skip_ansible_lint
+    - always

tasks/main.yml

@@ -50,6 +50,10 @@
   tags:
     - lxc_hosts-install
 
+- include: lxc_cache_prestage.yml
+  tags:
+    - lxc_hosts-config
+
 - include: lxc_post_install.yml
   tags:
     - lxc_hosts-config

templates/systemd-proxy-unit.conf.j2

@@ -1,6 +1,6 @@
 # {{ ansible_managed }}
 
 [Service]
-{% for key, value in lxc_cache_environment.items() %}
+{% for key, value in (deployment_environment_variables | default({})).items() %}
 Environment={{ key }}={{ value }}
 {% endfor %}

tests/test.yml

@@ -67,7 +67,7 @@
         - skip_ansible_lint
     - name: Set a fact to override lxc_image_cache_server value when in nodepool
       set_fact:
-        lxc_image_cache_server: "{{ lxc_reverse_proxy.stdout }}"
+        lxc_image_cache_server_mirrors: ["http://{{ lxc_reverse_proxy.stdout.strip('/') }}"]
       when:
         - nodepool.stat.exists | bool
 

vars/redhat-7.yml

@@ -18,6 +18,7 @@ systemd_utils_prefix: "/lib/systemd"
 
 # Required rpm packages.
 lxc_hosts_distro_packages:
+  - aria2
   - bridge-utils
   - btrfs-progs
   - dbus

vars/suse-42.yml

@@ -22,6 +22,7 @@ lxc_hosts_distro_packages:
   - apparmor-parser
   - apparmor-profiles
   - apparmor-utils
+  - aria2
   - bridge-utils
   - btrfsprogs
   - dnsmasq

vars/ubuntu-16.04.yml

@@ -20,6 +20,7 @@ cache_timeout: 600
 lxc_hosts_distro_packages:
   - apparmor
   - apparmor-utils
+  - aria2
   - bridge-utils
   - btrfs-tools
   - cgmanager