playbooks: roles: bifrost-ironic-install: Verify checksum for IPA files

Use the newly published checksum files to verify the integrity of the
downloaded IPA files. If verification fails, we try a few more times
before we give up since that would either mean that there is a
connectivity issue on our end or a problem with the uploaded files.
Checksum verification is enabled by default but it can be easily
disabled in case we do a non-upstream build with no checksum files
available.

Depends-On: I66afa3682ee1461ba1cdb68a134eaa4fd85af4e0
Change-Id: I7b3cad1b42e297b87ca839d66a16303250e27404

playbooks/roles/bifrost-ironic-install/defaults/main.yml

@@ -44,8 +44,12 @@ ipa_kernel: "{{http_boot_folder}}/ipa.vmlinuz"
 ipa_ramdisk: "{{http_boot_folder}}/ipa.initramfs"
 ipa_kernel_url: "{{ ipa_file_protocol }}://{{ hostvars[inventory_hostname]['ansible_' + ans_network_interface]['ipv4']['address'] }}:{{file_url_port}}/ipa.vmlinuz"
 ipa_kernel_upstream_url: https://tarballs.openstack.org/ironic-python-agent/tinyipa/files/tinyipa-master.vmlinuz
+ipa_kernel_upstream_checksum_algo: "sha256"
+ipa_kernel_upstream_checksum_url: "{{ ipa_kernel_upstream_url }}.{{ ipa_kernel_upstream_checksum_algo }}"
 ipa_ramdisk_url: "{{ ipa_file_protocol }}://{{ hostvars[inventory_hostname]['ansible_' + ans_network_interface]['ipv4']['address'] }}:{{file_url_port}}/ipa.initramfs"
 ipa_ramdisk_upstream_url: https://tarballs.openstack.org/ironic-python-agent/tinyipa/files/tinyipa-master.gz
+ipa_ramdisk_upstream_checksum_algo: "sha256"
+ipa_ramdisk_upstream_checksum_url: "{{ ipa_ramdisk_upstream_url }}.{{ ipa_ramdisk_upstream_checksum_algo }}"
 deploy_image_filename: "deployment_image.qcow2"
 deploy_image: "{{http_boot_folder}}/{{deploy_image_filename}}"
 # Use cirros instead of building an image via diskimage-builder

playbooks/roles/bifrost-ironic-install/tasks/download_ipa_image.yml

@@ -19,14 +19,60 @@
   stat: path={{ ipa_kernel }}
   register: test_ipa_kernel_present
 
+- block:
+  - name: "Download IPA kernel checksum file"
+    get_url: url="{{ ipa_kernel_upstream_checksum_url }}" dest="{{ ipa_kernel }}.{{ ipa_kernel_upstream_checksum_algo }}" timeout=300
+  - name: "Extract IPA kernel checksum"
+    shell: awk '/{{ ipa_kernel_upstream_url | basename }}/{print $1}' "{{ ipa_kernel }}.{{ ipa_kernel_upstream_checksum_algo }}"
+    register: parsed_ipa_kernel_checksum
+  - fail:
+      msg: "Failed to extract checksum for {{ ipa_kernel_upstream_url | basename }}"
+    when: parsed_ipa_kernel_checksum.stdout == ""
+  - set_fact:
+      ipa_kernel_checksum: "{{ ipa_kernel_upstream_checksum_algo }}:{{ parsed_ipa_kernel_checksum.stdout }}"
+  when: ipa_kernel_upstream_checksum_url != ""
+
 - name: "Download IPA kernel"
-  get_url: url={{ ipa_kernel_upstream_url }} dest={{ ipa_kernel }} timeout=300
+  get_url:
+    url: "{{ ipa_kernel_upstream_url }}"
+    dest: "{{ ipa_kernel }}"
+    checksum: "{{ ipa_kernel_checksum | default(omit) }}"
+    timeout: 300
+    # Keep downloading it until we get a good copy
+    force: yes
+  register: ipa_kernel_download_done
+  until: ipa_kernel_download_done|succeeded
+  retries: 5
+  delay: 10
   when: test_ipa_kernel_present.stat.exists == false
 
 - name: "Test if IPA image is present"
   stat: path={{ ipa_ramdisk }}
   register: test_ipa_image_present
 
+- block:
+  - name: "Download IPA image checksum"
+    get_url: url="{{ ipa_ramdisk_upstream_checksum_url }}" dest="{{ ipa_ramdisk }}.{{ ipa_ramdisk_upstream_checksum_algo }}" timeout=300
+  - name: "Extract IPA ramdisk checksum"
+    shell: awk '/{{ ipa_ramdisk_upstream_url | basename }}/{print $1}' "{{ ipa_ramdisk }}.{{ ipa_ramdisk_upstream_checksum_algo }}"
+    register: parsed_ipa_ramdisk_checksum
+  - fail:
+      msg: "Failed to extract checksum for {{ ipa_ramdisk_upstream_url | basename }}"
+    when: parsed_ipa_ramdisk_checksum.stdout == ""
+  - set_fact:
+      ipa_ramdisk_checksum: "{{ ipa_ramdisk_upstream_checksum_algo }}:{{ parsed_ipa_ramdisk_checksum.stdout }}"
+  when: ipa_ramdisk_upstream_checksum_url != ""
+
 - name: "Download IPA image"
-  get_url: url={{ ipa_ramdisk_upstream_url }} dest={{ ipa_ramdisk }} timeout=300
+  get_url:
+    url: "{{ ipa_ramdisk_upstream_url }}"
+    dest: "{{ ipa_ramdisk }}"
+    checksum: "{{ ipa_ramdisk_checksum | default(omit) }}"
+    timeout: 300
+    # Keep downloading it until we get a good copy
+    force: yes
+  register: ipa_ramdisk_download_done
+  until: ipa_ramdisk_download_done|succeeded
+  retries: 5
+  delay: 10
   when: test_ipa_image_present.stat.exists == false

releasenotes/notes/verify-checksum-ipa-a73cf936d0ae2ce1.yaml

@@ -0,0 +1,20 @@
+---
+features:
+  - Downloaded IPA files can now be verified using checksum files.
+    Upstream builds will be verified by default but you can disable
+    this behavior by setting ``ipa_kernel_upstream_checksum_url`` or
+    ``ipa_ramdisk_upstream_checksum_url`` variables to empty strings.
+    The default checksum algorithm is ``sha256`` which matches
+    the one provided in the upstream files. In case you want to
+    provide your own checksum files, you can set the previously mentioned
+    variables appropriately to match your setup. You can also set
+    ``ipa_kernel_upstream_checksum_algo`` or ``ipa_ramdisk_upstream_checksum_algo``
+    to checksum algorithms like ``md5`` in case you want to provide
+    non-sha256 checksums. Be careful though because these values
+    must be valid for Ansible ``get_url`` module's ``checksum``
+    parameter. Finally, it's also possible to provide the checksum
+    directly by setting the ``ipa_kernel_checksum`` or ``ipa_ramdisk_checksum``
+    variables to ``$algorithm:$checksum``.
+    In case the verification fails, bifrost will retry a few more times
+    to re-download and re-verify the files before giving up assuming
+    there is a network issue or a file corruption on the remote server.