Add support for enabling ipxe boot with ironic

When enable_ironic_ipxe is set in /etc/kolla/globals.yml,
the following happens:

- a new docker container, ironic_ipxe, is created. This contains
  an apache webserver used to serve up the boot images
- ironic is configured to use ipxe

Change-Id: I08fca1864a00afb768494406c49e968920c83ae7
Implements: blueprint ironic-ipxe

ansible/group_vars/all.yml

@@ -210,6 +210,7 @@ influxdb_http_port: "8086"
 
 ironic_api_port: "6385"
 ironic_inspector_port: "5050"
+ironic_ipxe_port: "8089"
 
 iscsi_port: "3260"
 
@@ -454,6 +455,7 @@ enable_horizon_zun: "{{ enable_zun | bool }}"
 enable_hyperv: "no"
 enable_influxdb: "{{ enable_monasca | bool }}"
 enable_ironic: "no"
+enable_ironic_ipxe: "no"
 enable_ironic_pxe_uefi: "no"
 enable_iscsid: "{{ (enable_cinder | bool and enable_cinder_backend_iscsi | bool) or enable_ironic | bool }}"
 enable_karbor: "no"

ansible/inventory/all-in-one

@@ -459,6 +459,9 @@ ironic
 [ironic-pxe:children]
 ironic
 
+[ironic-ipxe:children]
+ironic
+
 # Magnum
 [magnum-api:children]
 magnum

ansible/inventory/multinode

@@ -468,6 +468,9 @@ ironic
 [ironic-pxe:children]
 ironic
 
+[ironic-ipxe:children]
+ironic
+
 # Magnum
 [magnum-api:children]
 magnum

ansible/roles/ironic/defaults/main.yml

@@ -27,6 +27,7 @@ ironic_services:
       - "kolla_logs:/var/log/kolla"
       - "ironic:/var/lib/ironic"
       - "ironic_pxe:/tftpboot/"
+      - "ironic_ipxe:/httpboot/"
   ironic-inspector:
     container_name: ironic_inspector
     group: ironic-inspector
@@ -47,6 +48,16 @@ ironic_services:
       - "/etc/localtime:/etc/localtime:ro"
       - "ironic_pxe:/tftpboot/"
       - "kolla_logs:/var/log/kolla"
+  ironic-ipxe:
+    container_name: ironic_ipxe
+    group: ironic-ipxe
+    enabled: "{{ enable_ironic_ipxe | bool }}"
+    image: "{{ ironic_pxe_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/ironic-ipxe/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "ironic_ipxe:/httpboot/"
+      - "kolla_logs:/var/log/kolla"
   ironic-dnsmasq:
     container_name: ironic_dnsmasq
     group: ironic-inspector
@@ -125,6 +136,7 @@ openstack_ironic_inspector_auth: "{{ openstack_auth }}"
 
 ironic_dnsmasq_interface: "{{ api_interface }}"
 ironic_dnsmasq_dhcp_range:
-ironic_dnsmasq_boot_file: "pxelinux.0"
+ironic_dnsmasq_boot_file: "{% if enable_ironic_ipxe | bool %}undionly.kpxe{% else %}pxelinux.0{% endif %}"
 ironic_cleaning_network:
 ironic_console_serial_speed: "115200n8"
+ironic_ipxe_url: http://{{ api_interface_address }}:{{ ironic_ipxe_port }}

ansible/roles/ironic/handlers/main.yml

@@ -91,6 +91,28 @@
       or ironic_kernel.changed | bool
       or ironic_pxe_container.changed | bool
 
+- name: Restart ironic-ipxe container
+  vars:
+    service_name: "ironic-ipxe"
+    service: "{{ ironic_services[service_name] }}"
+    config_json: "{{ ironic_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    ironic_ipxe_container: "{{ check_ironic_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or ironic_ipxe_inspector_boot_script.changed | bool
+      or ironic_ipxe_apache_confs.changed | bool
+      or ironic_kernel_ipxe.changed | bool
+      or ironic_ipxe_container.changed | bool
+
 - name: Restart ironic-dnsmasq container
   vars:
     service_name: "ironic-dnsmasq"

ansible/roles/ironic/tasks/config.yml

@@ -123,6 +123,7 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
     - not enable_ironic_pxe_uefi | bool
+    - not enable_ironic_ipxe | bool
   notify:
     - Restart ironic-pxe container
 
@@ -146,7 +147,7 @@
   notify:
     - Restart ironic-pxe container
 
-- name: Copying ironic-agent kernel and initramfs
+- name: Copying ironic-agent kernel and initramfs (PXE)
   vars:
     service: "{{ ironic_services['ironic-pxe'] }}"
   copy:
@@ -165,9 +166,69 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
     - not enable_ironic_pxe_uefi | bool
+    - not enable_ironic_ipxe | bool
   notify:
     - Restart ironic-pxe container
 
+- name: Copying ironic-agent kernel and initramfs (iPXE)
+  vars:
+    service: "{{ ironic_services['ironic-ipxe'] }}"
+  copy:
+    src: "{{ node_custom_config }}/ironic/{{ item }}"
+    dest: "{{ node_config_directory }}/ironic-ipxe/{{ item }}"
+    mode: "0660"
+  become: true
+  register: ironic_kernel_ipxe
+  with_items:
+    - "ironic-agent.kernel"
+    - "ironic-agent.initramfs"
+  when:
+    # Only required when Ironic inspector is in use.
+    - groups['ironic-inspector'] | length > 0
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  notify:
+    - Restart ironic-ipxe container
+
+- name: Copying inspector.ipxe
+  vars:
+    service: "{{ ironic_services['ironic-ipxe'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/ironic-ipxe/inspector.ipxe"
+    mode: "0660"
+  become: true
+  register: ironic_ipxe_inspector_boot_script
+  with_first_found:
+    - "{{ node_custom_config }}/ironic/{{ inventory_hostname }}/inspector.ipxe"
+    - "{{ node_custom_config }}/ironic/inspector.ipxe"
+    - "inspector.ipxe.j2"
+  when:
+    # Only required when Ironic inspector is in use.
+    - groups['ironic-inspector'] | length > 0
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+  notify:
+    - Restart ironic-ipxe container
+
+- name: Copying iPXE apache config
+  vars:
+    service: "{{ ironic_services['ironic-ipxe'] }}"
+  template:
+    src: "{{ item }}"
+    dest: "{{ node_config_directory }}/ironic-ipxe/httpd.conf"
+    mode: "0660"
+  become: true
+  register: ironic_ipxe_apache_confs
+  with_first_found:
+    - "{{ node_custom_config }}/ironic/ironic-ipxe-httpd.conf"
+    - "ironic-ipxe-httpd.conf.j2"
+  when:
+    - service.enabled | bool
+    - inventory_hostname in groups[service.group]
+  notify:
+    - Restart ironic-ipxe container
+
 - name: Copying over existing policy file
   vars:
     services_require_policy_json:

ansible/roles/ironic/tasks/deploy.yml

@@ -8,7 +8,8 @@
   when: inventory_hostname in groups['ironic-api'] or
         inventory_hostname in groups['ironic-conductor'] or
         inventory_hostname in groups['ironic-inspector'] or
-        inventory_hostname in groups['ironic-pxe']
+        inventory_hostname in groups['ironic-pxe'] or
+        inventory_hostname in groups['ironic-ipxe']
 
 - include: bootstrap.yml
   when: inventory_hostname in groups['ironic-api'] or

ansible/roles/ironic/tasks/precheck.yml

@@ -4,6 +4,7 @@
     name:
       - ironic_api
       - ironic_inspector
+      - ironic_ipxe
   register: container_facts
 
 - name: Checking free port for Ironic API
@@ -28,6 +29,18 @@
     - container_facts['ironic_inspector'] is not defined
     - inventory_hostname in groups['ironic-inspector']
 
+- name: Checking free port for Ironic iPXE
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ ironic_ipxe_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - enable_ironic_ipxe | bool
+    - container_facts['ironic_ipxe'] is not defined
+    - inventory_hostname in groups['ironic-ipxe']
+
 - name: Checking ironic-agent files exist for Ironic Inspector
   local_action: stat path="{{ node_custom_config }}/ironic/{{ item }}"
   run_once: True
@@ -36,7 +49,8 @@
   when:
     # Only required when Ironic inspector is in use.
     - groups['ironic-inspector'] | length > 0
-    - inventory_hostname in groups['ironic-pxe']
+    - (not enable_ironic_ipxe | bool and inventory_hostname in groups['ironic-pxe']) or
+        (enable_ironic_ipxe | bool and inventory_hostname in groups['ironic-ipxe'])
     - not enable_ironic_pxe_uefi | bool
   with_items:
     - "ironic-agent.kernel"

ansible/roles/ironic/templates/inspector.ipxe.j2

@@ -0,0 +1,10 @@
+#!ipxe
+
+:retry_dhcp
+dhcp || goto retry_dhcp
+
+:retry_boot
+imgfree
+kernel --timeout 30000 {{ ironic_ipxe_url }}/ironic-agent.kernel ipa-inspection-callback-url=http://{{ kolla_internal_vip_address }}:{{ ironic_inspector_port }}/v1/continue systemd.journald.forward_to_console=yes BOOTIF=${mac} initrd=agent.ramdisk || goto retry_boot
+initrd --timeout 30000 {{ ironic_ipxe_url }}/ironic-agent.initramfs || goto retry_boot
+boot

ansible/roles/ironic/templates/ironic-conductor.json.j2

@@ -29,6 +29,11 @@
             "path": "/tftpboot",
             "owner": "ironic:ironic",
             "recurse": true
+        },
+        {
+            "path": "/httpboot",
+            "owner": "ironic:ironic",
+            "recurse": true
         }
     ]
 }

ansible/roles/ironic/templates/ironic-dnsmasq.conf.j2

@@ -5,5 +5,16 @@ dhcp-option=option:tftp-server,{{ api_interface_address }}
 dhcp-option=option:server-ip-address,{{ api_interface_address }}
 bind-interfaces
 dhcp-sequential-ip
-dhcp-option=option:bootfile-name,{{ ironic_dnsmasq_boot_file }}
 dhcp-option=210,/tftpboot/
+{% if enable_ironic_ipxe | bool %}
+dhcp-match=ipxe,175
+dhcp-match=set:efi,option:client-arch,7
+dhcp-match=set:efi,option:client-arch,9
+# Client is already running iPXE; move to next stage of chainloading
+dhcp-option=tag:ipxe,option:bootfile-name,{{ ironic_ipxe_url }}/inspector.ipxe
+# Client is PXE booting over EFI without iPXE ROM,
+# send EFI version of iPXE chainloader
+dhcp-option=tag:efi,tag:!ipxe,option:bootfile-name,ipxe.efi
+{% endif %}
+dhcp-option=option:bootfile-name,{{ ironic_dnsmasq_boot_file }}
+

ansible/roles/ironic/templates/ironic-ipxe-httpd.conf.j2

@@ -0,0 +1,16 @@
+Listen {{ api_interface_address }}:{{ ironic_ipxe_port }}
+
+TraceEnable off
+
+<VirtualHost *:{{ ironic_ipxe_port }}>
+    LogLevel warn
+    ErrorLog "/var/log/kolla/ironic/ironic-ipxe-error.log"
+    LogFormat "%h %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
+    CustomLog "/var/log/kolla/ironic/ironic-ipxe-access.log" logformat
+    DocumentRoot "/httpboot"
+    <Directory /httpboot>
+        Options FollowSymLinks
+        AllowOverride None
+        Require all granted
+    </Directory>
+</VirtualHost>

ansible/roles/ironic/templates/ironic-ipxe.json.j2

@@ -0,0 +1,33 @@
+{% set apache_conf_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %}
+{% set apache_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %}
+{
+    "command": "{{ apache_cmd }} -DFOREGROUND",
+    "config_files": [
+{% if groups['ironic-inspector'] | length > 0 %}
+        {
+            "source": "{{ container_config_directory }}/ironic-agent.kernel",
+            "dest": "/httpboot/ironic-agent.kernel",
+            "owner": "root",
+            "perm": "0644"
+        },
+        {
+            "source": "{{ container_config_directory }}/ironic-agent.initramfs",
+            "dest": "/httpboot/ironic-agent.initramfs",
+            "owner": "root",
+            "perm": "0644"
+        },
+        {
+            "source": "{{ container_config_directory }}/inspector.ipxe",
+            "dest": "/httpboot/inspector.ipxe",
+            "owner": "root",
+            "perm": "0644"
+        },
+{% endif %}
+        {
+            "source": "{{ container_config_directory }}/httpd.conf",
+            "dest": "/etc/{{ apache_conf_dir }}/httpboot.conf",
+            "owner": "root",
+            "perm": "0644"
+        }
+    ]
+}

ansible/roles/ironic/templates/ironic-pxe.json.j2

@@ -4,7 +4,7 @@
 {
     "command": "/usr/sbin/in.tftpd --verbose --foreground --user root --address 0.0.0.0:69 --map-file /map-file /tftpboot",
     "config_files": [
-{% if groups['ironic-inspector'] | length > 0 %}
+{% if not enable_ironic_ipxe | bool and groups['ironic-inspector'] | length > 0 %}
 {% if not enable_ironic_pxe_uefi | bool %}
         {
             "source": "{{ container_config_directory }}/ironic-agent.kernel",

ansible/roles/ironic/templates/ironic.conf.j2

@@ -97,6 +97,20 @@ deploy_logs_collect = always
 
 [pxe]
 pxe_append_params = nofb nomodeset vga=normal console=tty0 console=ttyS0,{{ ironic_console_serial_speed }}
+{% if enable_ironic_ipxe | bool %}
+ipxe_enabled = True
+pxe_bootfile_name = undionly.kpxe
+uefi_pxe_bootfile_name = ipxe.efi
+pxe_config_template = $pybasedir/drivers/modules/ipxe_config.template
+uefi_pxe_config_template = $pybasedir/drivers/modules/ipxe_config.template
+tftp_root = /httpboot
+tftp_master_path = /httpboot/master_images
+{% endif %}
+
+{% if enable_ironic_ipxe | bool %}
+[deploy]
+http_url = {{ ironic_ipxe_url }}
+{% endif %}
 
 [oslo_middleware]
 enable_proxy_headers_parsing = True

doc/source/reference/ironic-guide.rst

@@ -57,6 +57,39 @@ be used:
 
 .. end
 
+Enable iPXE booting (optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can optionally enable booting via iPXE by setting ``enable_ironic_ipxe`` to
+true in ``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+    enable_ironic_ipxe: "yes"
+
+.. end
+
+This will enable deployment of a docker container, called ironic_ipxe, running
+the web server which iPXE uses to obtain it's boot images.
+
+The port used for the iPXE webserver is controlled via ``ironic_ipxe_port`` in
+``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+    ironic_ipxe_port: "8089"
+
+.. end
+
+The following changes will occur if iPXE booting is enabled:
+
+- Ironic will be configured with the ``ipxe_enabled`` configuration option set
+  to true
+- The inspection ramdisk and kernel will be loaded via iPXE
+- The DHCP servers will be configured to chainload iPXE from an existing PXE
+  environment. You may also boot directly to iPXE by some other means e.g by
+  burning it to the option rom of your ethernet card.
+
 Deployment
 ~~~~~~~~~~
 Run the deploy as usual:

releasenotes/notes/support-ironic-ipxe-boot-2ea7f598748403bd.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Adds support for booting bare metal nodes with Ironic using iPXE.
+    This is enabled via the ``enable_ironic_ipxe`` flag.

tests/templates/inventory.j2

@@ -438,6 +438,9 @@ ironic
 [ironic-pxe:children]
 ironic
 
+[ironic-ipxe:children]
+ironic
+
 # Magnum
 [magnum-api:children]
 magnum