Merge "Fix IPA image download"

ansible/group_vars/all/bifrost

@@ -79,9 +79,21 @@ kolla_bifrost_inspector_deploy_ramdisk: "http://{{ provision_oc_net_name | net_i
 # URL of Ironic Python Agent (IPA) kernel image.
 kolla_bifrost_ipa_kernel_upstream_url: "{{ inspector_ipa_kernel_upstream_url }}"
 
+# URL of checksum of Ironic Python Agent (IPA) kernel image.
+kolla_bifrost_ipa_kernel_checksum_url: "{{ inspector_ipa_kernel_checksum_url }}"
+
+# Algorithm of checksum of Ironic Python Agent (IPA) kernel image.
+kolla_bifrost_ipa_kernel_checksum_algorithm: "{{ inspector_ipa_kernel_checksum_algorithm }}"
+
 # URL of Ironic Python Agent (IPA) ramdisk image.
 kolla_bifrost_ipa_ramdisk_upstream_url: "{{ inspector_ipa_ramdisk_upstream_url }}"
 
+# URL of checksum of Ironic Python Agent (IPA) ramdisk image.
+kolla_bifrost_ipa_ramdisk_checksum_url: "{{ inspector_ipa_ramdisk_checksum_url }}"
+
+# Algorithm of checksum of Ironic Python Agent (IPA) ramdisk image.
+kolla_bifrost_ipa_ramdisk_checksum_algorithm: "{{ inspector_ipa_ramdisk_checksum_algorithm }}"
+
 ###############################################################################
 # Inventory configuration.
 

ansible/group_vars/all/inspector

@@ -8,9 +8,21 @@ inspector_extra_kernel_options: "{{ ipa_kernel_options }}"
 # URL of Ironic Python Agent (IPA) kernel image.
 inspector_ipa_kernel_upstream_url: "{{ ipa_kernel_upstream_url }}"
 
+# URL of checksum of Ironic Python Agent (IPA) kernel image.
+inspector_ipa_kernel_checksum_url: "{{ ipa_kernel_checksum_url }}"
+
+# Algorithm of checksum of Ironic Python Agent (IPA) kernel image.
+inspector_ipa_kernel_checksum_algorithm: "{{ ipa_kernel_checksum_algorithm }}"
+
 # URL of Ironic Python Agent (IPA) ramdisk image.
 inspector_ipa_ramdisk_upstream_url: "{{ ipa_ramdisk_upstream_url }}"
 
+# URL of checksum of Ironic Python Agent (IPA) ramdisk image.
+inspector_ipa_ramdisk_checksum_url: "{{ ipa_ramdisk_checksum_url }}"
+
+# Algorithm of checksum of Ironic Python Agent (IPA) ramdisk image.
+inspector_ipa_ramdisk_checksum_algorithm: "{{ ipa_ramdisk_checksum_algorithm }}"
+
 ###############################################################################
 # Ironic inspector processing configuration.
 

ansible/group_vars/all/ipa

@@ -65,12 +65,24 @@ ipa_images_kernel_name: "ipa.vmlinuz"
 # URL of Ironic deployment kernel image to download.
 ipa_kernel_upstream_url: "https://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe-stable-queens.vmlinuz"
 
+# URL of checksum of Ironic deployment kernel image.
+ipa_kernel_checksum_url: "{{ ipa_kernel_upstream_url }}.{{ ipa_kernel_checksum_algorithm }}"
+
+# Algorithm of checksum of Ironic deployment kernel image.
+ipa_kernel_checksum_algorithm: "sha256"
+
 # Name of Ironic deployment ramdisk image to register in Glance.
 ipa_images_ramdisk_name: "ipa.initramfs"
 
 # URL of Ironic deployment ramdisk image to download.
 ipa_ramdisk_upstream_url: "https://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe_image-oem-stable-queens.cpio.gz"
 
+# URL of checksum of Ironic deployment ramdisk image.
+ipa_ramdisk_checksum_url: "{{ ipa_ramdisk_upstream_url }}.{{ ipa_ramdisk_checksum_algorithm }}"
+
+# Algorithm of checksum of Ironic deployment ramdisk image.
+ipa_ramdisk_checksum_algorithm: "sha256"
+
 ###############################################################################
 # Ironic Python Agent (IPA) deployment configuration.
 

ansible/kolla-openstack.yml

@@ -170,7 +170,11 @@
             - name: Set facts containing IPA kernel and ramdisk URLs
               set_fact:
                 kolla_inspector_ipa_kernel_upstream_url: "{{ inspector_ipa_kernel_upstream_url }}"
+                kolla_inspector_ipa_kernel_checksum_url: "{{ inspector_ipa_kernel_checksum_url }}"
+                kolla_inspector_ipa_kernel_checksum_algorithm: "{{ inspector_ipa_kernel_checksum_algorithm }}"
                 kolla_inspector_ipa_ramdisk_upstream_url: "{{ inspector_ipa_ramdisk_upstream_url }}"
+                kolla_inspector_ipa_ramdisk_checksum_url: "{{ inspector_ipa_ramdisk_checksum_url }}"
+                kolla_inspector_ipa_ramdisk_checksum_algorithm: "{{ inspector_ipa_ramdisk_checksum_algorithm }}"
               when: not ipa_build_images | bool
 
             - name: Set facts containing IPA kernel and ramdisk paths

ansible/overcloud-ipa-images.yml

@@ -58,7 +58,11 @@
       set_fact:
         # Don't pass the kernel and ramdisk image URLs if using built images.
         ipa_images_kernel_url: "{{ ipa_kernel_upstream_url }}"
+        ipa_images_kernel_checksum_url: "{{ ipa_kernel_checksum_url }}"
+        ipa_images_kernel_checksum_algorithm: "{{ ipa_kernel_checksum_algorithm }}"
         ipa_images_ramdisk_url: "{{ ipa_ramdisk_upstream_url }}"
+        ipa_images_ramdisk_checksum_url: "{{ ipa_ramdisk_checksum_url }}"
+        ipa_images_ramdisk_checksum_algorithm: "{{ ipa_ramdisk_checksum_algorithm }}"
       when: not ipa_build_images | bool
 
     - name: Check whether the image cache directory exists

ansible/roles/image-download/defaults/main.yml

@@ -0,0 +1,19 @@
+---
+# URL of the image.
+# Mutually exclusive with image_download_path.
+image_download_url:
+
+# URL of a checksum of the image.
+# Mutually exclusive with image_download_path.
+image_download_checksum_url:
+
+# Algorithm of a checksum of the image.
+# Mutually exclusive with image_download_path.
+image_download_checksum_algorithm:
+
+# Path to a local file containing the image.
+# Mutually exclusive with image_download_url.
+image_download_path:
+
+# Path to the image's destination.
+image_download_dest:

ansible/roles/image-download/tasks/main.yml

@@ -0,0 +1,42 @@
+---
+- block:
+    - block:
+        - name: Fail if the checksum algorithm is not set
+          fail:
+            msg: "Checksum algorithm for image {{ image_download_url }} not set"
+          when: image_download_checksum_algorithm is none or
+                image_download_checksum_algorithm == ""
+
+        - name: Get the expected checksum
+          uri:
+            url: "{{ image_download_checksum_url }}"
+            return_content: true
+          register: expected_checksum
+      when:
+        - image_download_checksum_url is not none
+        - image_download_checksum_url != ""
+
+    - name: Ensure the image is downloaded
+      vars:
+        checksum: "{{ image_download_checksum_algorithm }}:{{ expected_checksum.content.split(' ')[0] }}"
+      get_url:
+        url: "{{ image_download_url }}"
+        dest: "{{ image_download_dest }}"
+        mode: 0640
+        # If the file exists locally, its checksum will be compared with this.
+        checksum: "{{ checksum if expected_checksum is not skipped else omit }}"
+        # Always download the image if we have no checksum to compare with.
+        force: "{{ expected_checksum is skipped }}"
+        backup: true
+  when:
+    - image_download_url is not none
+    - image_download_url != ""
+
+- name: Ensure the local image is copied
+  copy:
+    src: "{{ image_download_path }}"
+    dest: "{{ image_download_dest }}"
+    mode: 0640
+  when:
+    - image_download_path is not none
+    - image_download_path != ""

ansible/roles/ipa-images/defaults/main.yml

@@ -24,6 +24,12 @@ ipa_images_kernel_name:
 # image in ipa_images_cache_path will be used.
 ipa_images_kernel_url:
 
+# URL of checksum of Ironic deployment kernel image.
+ipa_images_kernel_checksum_url:
+
+# Algorithm of checksum of Ironic deployment kernel image.
+ipa_images_kernel_checksum_algorithm:
+
 # Name of Ironic deployment ramdisk image to register in Glance.
 ipa_images_ramdisk_name:
 
@@ -31,6 +37,12 @@ ipa_images_ramdisk_name:
 # image in ipa_images_cache_path will be used.
 ipa_images_ramdisk_url:
 
+# URL of checksum of Ironic deployment ramdisk image.
+ipa_images_ramdisk_checksum_url:
+
+# Algorithm of checksum of Ironic deployment ramdisk image.
+ipa_images_ramdisk_checksum_algorithm:
+
 # Ansible host pattern for limiting which nodes are updated with deploy_ramdisk
 # and deploy_kernel properties
 ipa_images_compute_node_limit: baremetal-compute

ansible/roles/ipa-images/tasks/main.yml

@@ -7,18 +7,26 @@
     group: "{{ ansible_user_gid }}"
   become: True
 
-- name: Ensure Ironic Python Agent (IPA) images are downloaded
+- name: Ensure Ironic Python Agent (IPA) images are present
-  get_url:
+  vars:
-    url: "{{ item.url }}"
+    image_download_url: "{{ item.url }}"
-    dest: "{{ ipa_images_cache_path }}/{{ item.filename }}"
+    image_download_checksum_url: "{{ item.checksum_url }}"
-    force: true
+    image_download_checksum_algorithm: "{{ item.checksum_algorithm }}"
-    backup: true
+    image_download_dest: "{{ item.dest }}"
+  include_role:
+    name: image-download
   with_items:
     - url: "{{ ipa_images_kernel_url }}"
-      filename: "{{ ipa_images_kernel_name }}"
+      checksum_url: "{{ ipa_images_kernel_checksum_url }}"
+      checksum_algorithm: "{{ ipa_images_kernel_checksum_algorithm }}"
+      dest: "{{ ipa_images_cache_path }}/{{ ipa_images_kernel_name }}"
     - url: "{{ ipa_images_ramdisk_url }}"
-      filename: "{{ ipa_images_ramdisk_name }}"
+      checksum_url: "{{ ipa_images_ramdisk_checksum_url }}"
-  when: item.url != None
+      checksum_algorithm: "{{ ipa_images_ramdisk_checksum_algorithm }}"
+      dest: "{{ ipa_images_cache_path }}/{{ ipa_images_ramdisk_name }}"
+  when: item.url is not none
+  loop_control:
+    label: "{{ item.dest }}"
 
 - name: Compute the MD5 checksum of the Ironic Python Agent (IPA) images
   stat:

ansible/roles/kolla-bifrost/defaults/main.yml

@@ -58,9 +58,21 @@ kolla_bifrost_download_ipa: true
 # URL of Ironic Python Agent (IPA) kernel image.
 kolla_bifrost_ipa_kernel_upstream_url:
 
+# URL of checksum of Ironic Python Agent (IPA) kernel image.
+kolla_bifrost_ipa_kernel_checksum_url:
+
+# Algorithm of checksum of Ironic Python Agent (IPA) kernel image.
+kolla_bifrost_ipa_kernel_checksum_algorithm:
+
 # URL of Ironic Python Agent (IPA) ramdisk image.
 kolla_bifrost_ipa_ramdisk_upstream_url:
 
+# URL of checksum of Ironic Python Agent (IPA) ramdisk image.
+kolla_bifrost_ipa_ramdisk_checksum_url:
+
+# Algorithm of checksum of Ironic Python Agent (IPA) ramdisk image.
+kolla_bifrost_ipa_ramdisk_checksum_algorithm:
+
 # Server inventory to be configured in {{ kolla_node_custom_config_path }}/bifrost/servers.yml.
 kolla_bifrost_servers: {}
 

ansible/roles/kolla-bifrost/templates/bifrost.yml.j2

@@ -51,11 +51,31 @@ download_ipa: "{{ kolla_bifrost_download_ipa }}"
 ipa_kernel_upstream_url: "{{ kolla_bifrost_ipa_kernel_upstream_url }}"
 {% endif %}
 
+{% if kolla_bifrost_ipa_kernel_checksum_url %}
+# URL of checksum of Ironic Python Agent (IPA) kernel image.
+ipa_kernel_upstream_checksum_url: "{{ kolla_bifrost_ipa_kernel_checksum_url }}"
+{% endif %}
+
+{% if kolla_bifrost_ipa_kernel_checksum_algorithm %}
+# Algorithm of checksum of Ironic Python Agent (IPA) kernel image.
+ipa_kernel_upstream_checksum_algo: "{{ kolla_bifrost_ipa_kernel_checksum_algorithm }}"
+{% endif %}
+
 {% if kolla_bifrost_ipa_ramdisk_upstream_url %}
 # URL of Ironic Python Agent (IPA) ramdisk image.
 ipa_ramdisk_upstream_url: "{{ kolla_bifrost_ipa_ramdisk_upstream_url }}"
 {% endif %}
 
+{% if kolla_bifrost_ipa_ramdisk_checksum_url %}
+# URL of checksum of Ironic Python Agent (IPA) ramdisk image.
+ipa_ramdisk_upstream_checksum_url: "{{ kolla_bifrost_ipa_ramdisk_checksum_url }}"
+{% endif %}
+
+{% if kolla_bifrost_ipa_ramdisk_checksum_algorithm %}
+# Algorithm of checksum of Ironic Python Agent (IPA) ramdisk image.
+ipa_ramdisk_upstream_checksum_algo: "{{ kolla_bifrost_ipa_ramdisk_checksum_algorithm }}"
+{% endif %}
+
 {% if kolla_bifrost_extra_globals %}
 ###############################################################################
 # Extra configuration

ansible/roles/kolla-openstack/defaults/main.yml

@@ -224,10 +224,26 @@ kolla_inspector_extra_kernel_options: []
 # Mutually exclusive with kolla_inspector_ipa_kernel_path.
 kolla_inspector_ipa_kernel_upstream_url:
 
+# URL of checksum of Ironic Python Agent (IPA) kernel image for Ironic
+# inspector. Mutually exclusive with kolla_inspector_ipa_kernel_path.
+kolla_inspector_ipa_kernel_checksum_url:
+
+# Algorithm of checksum of Ironic Python Agent (IPA) kernel image for Ironic
+# inspector. Mutually exclusive with kolla_inspector_ipa_kernel_path.
+kolla_inspector_ipa_kernel_checksum_algorithm:
+
 # URL of Ironic Python Agent (IPA) ramdisk image for Ironic Inspector.
 # Mutually exclusive with kolla_inspector_ipa_ramdisk_path.
 kolla_inspector_ipa_ramdisk_upstream_url:
 
+# URL of checksum of Ironic Python Agent (IPA) ramdisk image for Ironic
+# Inspector. Mutually exclusive with kolla_inspector_ipa_ramdisk_path.
+kolla_inspector_ipa_ramdisk_checksum_url:
+
+# Algorithm of checksum of Ironic Python Agent (IPA) ramdisk image for Ironic
+# Inspector. Mutually exclusive with kolla_inspector_ipa_ramdisk_path.
+kolla_inspector_ipa_ramdisk_checksum_algorithm:
+
 # Path to Ironic Python Agent (IPA) kernel image for Ironic Inspector.
 # Mutually exclusive with kolla_inspector_ipa_kernel_upstream_url.
 kolla_inspector_ipa_kernel_path:

ansible/roles/kolla-openstack/tasks/config.yml

@@ -35,29 +35,29 @@
     - { src: zookeeper.cfg.j2, dest: zookeeper.cfg, enabled: "{{ kolla_enable_zookeeper }}" }
   when: item.enabled | bool
 
-- name: Ensure the ironic inspector kernel and ramdisk are downloaded
+- name: Ensure ironic inspector kernel and ramdisk images are present
-  get_url:
+  vars:
-    url: "{{ item.url }}"
+    image_download_url: "{{ item.url }}"
-    dest: "{{ kolla_node_custom_config_path }}/ironic/{{ item.dest }}"
+    image_download_checksum_url: "{{ item.checksum_url }}"
-    mode: 0640
+    image_download_checksum_algorithm: "{{ item.checksum_algorithm }}"
+    image_download_path: "{{ item.path }}"
+    image_download_dest: "{{ item.dest }}"
+  include_role:
+    name: image-download
   with_items:
-    - { url: "{{ kolla_inspector_ipa_kernel_upstream_url }}", dest: "ironic-agent.kernel" }
+    - url: "{{ kolla_inspector_ipa_kernel_upstream_url }}"
-    - { url: "{{ kolla_inspector_ipa_ramdisk_upstream_url }}", dest: "ironic-agent.initramfs" }
+      checksum_url: "{{ kolla_inspector_ipa_kernel_checksum_url }}"
-  when:
+      checksum_algorithm: "{{ kolla_inspector_ipa_kernel_checksum_algorithm }}"
-    - kolla_enable_ironic | bool
+      path: "{{ kolla_inspector_ipa_kernel_path }}"
-    - item.url != None
+      dest: "{{ kolla_node_custom_config_path }}/ironic/ironic-agent.kernel"
-
+    - url: "{{ kolla_inspector_ipa_ramdisk_upstream_url }}"
-- name: Ensure the ironic inspector kernel and ramdisk are copied
+      checksum_url: "{{ kolla_inspector_ipa_ramdisk_checksum_url }}"
-  copy:
+      checksum_algorithm: "{{ kolla_inspector_ipa_ramdisk_checksum_algorithm }}"
-    src: "{{ item.path }}"
+      path: "{{ kolla_inspector_ipa_ramdisk_path }}"
-    dest: "{{ kolla_node_custom_config_path }}/ironic/{{ item.dest }}"
+      dest: "{{ kolla_node_custom_config_path }}/ironic/ironic-agent.initramfs"
-    mode: 0640
+  when: kolla_enable_ironic | bool
-  with_items:
+  loop_control:
-    - { path: "{{ kolla_inspector_ipa_kernel_path }}", dest: "ironic-agent.kernel" }
+    label: "{{ item.dest }}"
-    - { path: "{{ kolla_inspector_ipa_ramdisk_path }}", dest: "ironic-agent.initramfs" }
-  when:
-    - kolla_enable_ironic | bool
-    - item.path != None
 
 # We support a fairly flexible mechanism of dropping config file templates into
 # an 'extra' config directory, and passing these through to kolla-ansible. We

doc/source/upgrading.rst

@@ -145,9 +145,17 @@ Upgrading Ironic Deployment Images
 
 Prior to upgrading the OpenStack control plane you should upgrade
 the deployment images. If you are using prebuilt images, update
-``ipa_kernel_upstream_url`` and ``ipa_ramdisk_upstream_url`` in
+the following variables in ``etc/kayobe/ipa.yml`` accordingly:
-``etc/kayobe/ipa.yml``, alternatively, you can update the files that the URLs
+
-point to. If building the images locally, follow the process outlined in
+* ``ipa_kernel_upstream_url``
+* ``ipa_kernel_checksum_url``
+* ``ipa_kernel_checksum_algorithm``
+* ``ipa_ramdisk_upstream_url``
+* ``ipa_ramdisk_checksum_url``
+* ``ipa_ramdisk_checksum_algorithm``
+
+Alternatively, you can update the files that the URLs point to. If building the
+images locally, follow the process outlined in
 :ref:`building_ironic_deployment_images`.
 
 To get Ironic to use an updated set of overcloud deployment images, you can run::

etc/kayobe/bifrost.yml

@@ -69,6 +69,27 @@
 # Ironic inspector deployment ramdisk location.
 #kolla_bifrost_inspector_deploy_ramdisk:
 
+###############################################################################
+# Ironic Python Agent (IPA) configuration.
+
+# URL of Ironic Python Agent (IPA) kernel image.
+#kolla_bifrost_ipa_kernel_upstream_url:
+
+# URL of checksum of Ironic Python Agent (IPA) kernel image.
+#kolla_bifrost_ipa_kernel_checksum_url:
+
+# Algorithm of checksum of Ironic Python Agent (IPA) kernel image.
+#kolla_bifrost_ipa_kernel_checksum_algorithm:
+
+# URL of Ironic Python Agent (IPA) ramdisk image.
+#kolla_bifrost_ipa_ramdisk_upstream_url:
+
+# URL of checksum of Ironic Python Agent (IPA) ramdisk image.
+#kolla_bifrost_ipa_ramdisk_checksum_url:
+
+# Algorithm of checksum of Ironic Python Agent (IPA) ramdisk image.
+#kolla_bifrost_ipa_ramdisk_checksum_algorithm:
+
 ###############################################################################
 # Inventory configuration.
 

etc/kayobe/inspector.yml

@@ -8,9 +8,21 @@
 # URL of Ironic Python Agent (IPA) kernel image.
 #inspector_ipa_kernel_upstream_url:
 
+# URL of checksum of Ironic Python Agent (IPA) kernel image.
+#inspector_ipa_kernel_checksum_url:
+
+# Algorithm of checksum of Ironic Python Agent (IPA) kernel image.
+#inspector_ipa_kernel_checksum_algorithm:
+
 # URL of Ironic Python Agent (IPA) ramdisk image.
 #inspector_ipa_ramdisk_upstream_url:
 
+# URL of checksum of Ironic Python Agent (IPA) ramdisk image.
+#inspector_ipa_ramdisk_checksum_url:
+
+# Algorithm of checksum of Ironic Python Agent (IPA) ramdisk image.
+#inspector_ipa_ramdisk_checksum_algorithm:
+
 ###############################################################################
 # Ironic inspector processing configuration.
 

etc/kayobe/ipa.yml

@@ -56,12 +56,24 @@
 # URL of Ironic deployment kernel image to download.
 #ipa_kernel_upstream_url:
 
+# URL of checksum of Ironic deployment kernel image.
+#ipa_kernel_checksum_url:
+
+# Algorithm of checksum of Ironic deployment kernel image.
+#ipa_kernel_checksum_algorithm:
+
 # Name of Ironic deployment ramdisk image to register in Glance.
 #ipa_images_ramdisk_name:
 
 # URL of Ironic deployment ramdisk image to download.
 #ipa_ramdisk_upstream_url:
 
+# URL of checksum of Ironic deployment ramdisk image.
+#ipa_ramdisk_checksum_url:
+
+# Algorithm of checksum of Ironic deployment ramdisk image.
+#ipa_ramdisk_checksum_algorithm:
+
 ###############################################################################
 # Ironic Python Agent (IPA) deployment configuration.
 

releasenotes/notes/fix-ipa-image-download-3f90f0f40d0feafd.yaml

@@ -0,0 +1,36 @@
+---
+upgrade:
+  - |
+    It is now possible to specify a URL of a file containing a checksum of the
+    Ironic Python Agent (IPA) images for the seed and overcloud Ironic and
+    Ironic Inspector services. This allows Kayobe to detect changes in the
+    image content and download as necessary. If specifying IPA images via URL,
+    the checksums should be configured accordingly.
+
+    The checksum URLs and algorithms are configured via these variables:
+
+    * ``{{ ipa_kernel_checksum_url }}``
+    * ``{{ ipa_kernel_checksum_algorithm }}``
+    * ``{{ ipa_ramdisk_checksum_url }}``
+    * ``{{ ipa_ramdisk_checksum_algorithm }}``
+
+    For the seed this may be customised via these variables:
+
+    * ``{{ kolla_bifrost_ipa_kernel_checksum_url }}``
+    * ``{{ kolla_bifrost_ipa_kernel_checksum_algorithm }}``
+    * ``{{ kolla_bifrost_ipa_ramdisk_checksum_url }}``
+    * ``{{ kolla_bifrost_ipa_ramdisk_checksum_algorithm }}``
+
+    For the overcloud Ironic Inspector service this may be customised via these
+    variables:
+
+    * ``{{ inspector_ipa_kernel_checksum_url }}``
+    * ``{{ inspector_ipa_kernel_checksum_algorithm }}``
+    * ``{{ inspector_ipa_ramdisk_checksum_url }}``
+    * ``{{ inspector_ipa_ramdisk_checksum_algorithm }}``
+fixes:
+  - |
+    Fixes an issue with downloading Ironic Python Agent (IPA) images where new
+    images would not be downloaded if the image had been downloaded previously.
+    See `Story 2001660 <https://storyboard.openstack.org/#!/story/2001660>`__
+    for details.