Use the deployment host to download images for tempest

Currently there are two sets of image downloads happening,
which is a common cause of failure in jobs due to the fact
that they're downloaded from a URL outside of OpenStack-CI.

This patch consolidates them into a single single download,
but also uses the deploy host to do the download. The download
gets done to the same location as where the image is cached
in OpenStack-CI, so as long as the SHA256SUM matches, it
will skip the download - making our tests faster and more
reliable.

As part of this, some legacy and unnecessary variables have
been removed. Also, we were uploading the same image twice
into Glance which is unnecessary. Instead we re-use the same
image ID twice in the tempest configuration.

Change-Id: I8d0afd08f6c80594d69b8d711261ae6365fad917

defaults/main.yml

@@ -174,16 +174,24 @@ tempest_pip_packages:
   - tempest
   - testscenarios
 
-# Please update SHA in tempest_images below when changing the cirros version.
-cirros_version: 0.3.5
-tempest_img_url: "http://download.cirros-cloud.net/{{ cirros_version }}/cirros-{{ cirros_version }}-x86_64-disk.img"
-tempest_image_dir: "/opt/images"
+# The list of images for tempest to download.
+# url: where to download from (required)
+# checksum: the checksum of the downloaded file to validate against, format: <algorithm>:<checksum> (optional)
+# format: format to use when uploading to glance (required)
+# name: name to use when uploading to glance - default is to use the downloaded file's name (optional)
 tempest_images:
-  - url: "{{ tempest_img_url }}"
-    sha256: "e137062a4dfbb4c225971b67781bc52183d14517170e16a3841d16f962ae7470"
-tempest_image_file: "cirros-{{ cirros_version }}-x86_64-disk.img"
-tempest_img_disk_format: qcow2
-tempest_img_name: 'cirros'
+  - url: "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img"
+    checksum: "sha256:e137062a4dfbb4c225971b67781bc52183d14517170e16a3841d16f962ae7470"
+    format: "qcow2"
+    name: "cirros"
+
+# The location where images are downloaded to
+tempest_image_dir: "/opt/cache/files"
+
+# Where the download is executed from.
+# Options are ['deployment-host', 'target-host']
+tempest_image_downloader: "deployment-host"
+
 tempest_enable_instance_password: True
 
 tempest_flavors:

releasenotes/notes/remove-duplicated-download-99a9ec5bfe4ba749.yaml

@@ -0,0 +1,37 @@
+---
+features:
+  - |
+    The ``tempest_images`` data structure for the ``os_tempest`` role
+    now expects the values for each image to include ``name`` (optionally)
+    and ``format`` (the disk format). Also, the optional variable ``checksum``
+    may be used to set the checksum expected for the file in the format
+    ``<algorithm>:<checksum>``.
+  - |
+    The default location for the image downloads in the ``os_tempest``
+    role set by the ``tempest_image_dir`` variable has now been changed
+    to be ``/opt/cache/files`` in order to match the default location
+    in nodepool. This improves the reliability of CI testing in
+    OpenStack CI as it will find the file already cached there.
+  - |
+    A new variable has been introduced into the ``os_tempest`` role
+    named ``tempest_image_downloader``. When set to ``deployment-host``
+    (which is the default) it uses the deployment host to handle the
+    download of images to be used for tempest testing. The images are
+    then uploaded to the target host for uploading into Glance.
+deprecations:
+  - |
+    The following variables have been removed from the ``os_tempest``
+    role to simplify it. They have been replaced through the use of
+    the data structure ``tempest_images`` which now has equivalent
+    variables per image.
+    - cirros_version
+    - tempest_img_url
+    - tempest_image_file
+    - tempest_img_disk_format
+    - tempest_img_name
+    - tempest_images.sha256 (replaced by checksum)
+fixes:
+  - |
+    The ``os_tempest`` tempest role was downloading images twice - once
+    arbitrarily, and once to use for testing. This has been consolidated
+    into a single download to a consistent location.

tasks/tempest_post_install.yml

@@ -50,18 +50,6 @@
     - path: "{{ tempest_image_dir }}"
   when: "{{ item.when | default(True) }}"
 
-- name: Image(s) download
-  get_url:
-    url: "{{ item.url }}"
-    dest: "{{ tempest_image_dir }}"
-    sha256sum: "{{ item.sha256 }}"
-  with_items: "{{ tempest_images }}"
-  when: tempest_service_available_glance | bool
-  register: fetch_url
-  until: fetch_url | success
-  retries: 6
-  delay: 5
-
 - name: Copy tempest config
   config_template:
     src: "tempest.conf.j2"

tasks/tempest_resources.yml

@@ -13,22 +13,52 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Download tempest image
-  get_url:
-    url: "{{ tempest_img_url }}"
-    dest: "/var/tmp/{{ tempest_img_url | basename }}"
-  when: tempest_service_available_glance | bool
+- name: Create deployment-host tempest_image_dir
+  file:
+    path: "{{ tempest_image_dir }}"
+    state: directory
+  delegate_to: localhost
+  when:
+    - tempest_service_available_glance | bool
+    - tempest_image_downloader == "deployment-host"
 
-- name: Ensure tempest image
+- name: Image(s) download
+  get_url:
+    url: "{{ item.url }}"
+    dest: "{{ tempest_image_dir }}/"
+    checksum: "{{ item.checksum | default(omit) }}"
+  with_items: "{{ tempest_images }}"
+  when: tempest_service_available_glance | bool
+  register: fetch_url
+  until: fetch_url | success
+  retries: 6
+  delay: 5
+  delegate_to: "{{ (tempest_image_downloader == 'deployment-host') | ternary('localhost', omit) }}"
+
+- name: Copy download images from deployment-host to target-host
+  copy:
+    src: "{{ tempest_image_dir }}/{{ item.url | basename }}"
+    dest: "{{ tempest_image_dir }}/"
+  with_items: "{{ tempest_images }}"
+  when:
+    - tempest_service_available_glance | bool
+    - tempest_image_downloader == "deployment-host"
+  register: fetch_url
+  until: fetch_url | success
+  retries: 6
+  delay: 5
+
+- name: Upload tempest images to glance
   os_image:
     cloud: default
     endpoint_type: internal
     validate_certs: "{{ keystone_service_internaluri_insecure | ternary(false, true) }}"
-    name: "{{ tempest_img_name }}"
-    filename: "/var/tmp/{{ tempest_img_url | basename }}"
+    name: "{{ item.name | default(item.url | basename) }}"
+    filename: "{{ tempest_image_dir }}/{{ item.url | basename }}"
     container_format: bare
-    disk_format: "{{ tempest_img_disk_format }}"
+    disk_format: "{{ item.format }}"
     is_public: True
+  with_items: "{{ tempest_images }}"
   register: tempest_image_create
   until: tempest_image_create | success
   retries: 5
@@ -39,26 +69,7 @@
 # set above to ensure the template will parse correctly.
 - name: Store tempest image id
   set_fact:
-    tempest_glance_image_id: "{{ tempest_service_available_glance | ternary(tempest_image_create.id, '') }}"
-
-- name: Ensure alt tempest image
-  os_image:
-    cloud: default
-    endpoint_type: internal
-    validate_certs: "{{ keystone_service_internaluri_insecure | ternary(false, true) }}"
-    name: "{{ tempest_img_name }}_alt"
-    filename: "/var/tmp/{{ tempest_img_url | basename }}"
-    container_format: bare
-    disk_format: "{{ tempest_img_disk_format }}"
-    is_public: True
-  register: tempest_image_alt_create
-  when: tempest_service_available_glance | bool
-
-# This fact is used in tempest.conf.j2; we set an empty string if it doesn't get
-# set above to ensure the template will parse correctly.
-- name: Store alt tempest image id
-  set_fact:
-    tempest_glance_image_alt_id: "{{ tempest_service_available_glance | ternary(tempest_image_alt_create.id, '') }}"
+    tempest_glance_image_id: "{{ tempest_service_available_glance | ternary(tempest_image_create['results'][0]['id'], '') }}"
 
 - name: Ensure tempest tenants
   keystone:

templates/tempest.conf.j2

@@ -24,7 +24,7 @@ endpoint_type = internalURL
 
 [compute]
 image_ref = {{ tempest_glance_image_id }}
-image_ref_alt = {{ tempest_glance_image_alt_id }}
+image_ref_alt = {{ tempest_glance_image_id }}
 flavor_ref = 201
 flavor_ref_alt = 202
 fixed_network_name = private
@@ -71,7 +71,7 @@ trust = false
 
 [image]
 endpoint_type = internalURL
-http_image = {{ tempest_img_url }}
+http_image = {{ tempest_images[0]['url'] }}
 
 
 [image-feature-enabled]
@@ -124,9 +124,9 @@ lock_path = {{ tempest_venv_bin | dirname }}/locks
 
 [scenario]
 img_dir = {{ tempest_image_dir }}
-img_file = {{ tempest_image_file }}
+img_file = {{ tempest_images[0]['url'] | basename }}
 img_container_format = bare
-img_disk_format = {{ tempest_img_disk_format }}
+img_disk_format = {{ tempest_images[0]['format'] }}
 
 
 [service_available]