Add support for bootstrapping iDRACs on Dell servers

The idrac-bootstrap.yml playbook is intended to bootstrap network configuration
of the iDRACs on a set of Dell servers. By default, iDRACs have a known static
IP address (192.168.0.120) and a set of default credentials configured. Since
all iDRACs have the same default IP address, we need a way to isolate a single
iDRAC while we set its network configuration. We do this using a temporary VLAN
accessible from one of the controllers.

ansible/group_vars/all/idrac

@@ -0,0 +1,43 @@
+---
+###############################################################################
+# iDRAC configuration.
+
+# Default username for iDRACs.
+idrac_default_username: root
+
+# Default password for iDRACs.
+idrac_default_password: calvin
+
+# Default IP address for iDRACs.
+idrac_default_ip: 192.168.0.120
+
+# Dict mapping host description (as found in switch interface configuration) to
+# IP address of the iDRAC for that host.
+idrac_network_ips:
+
+# Gateway IP address for iDRAC network.
+idrac_network_gateway:
+
+# IP netmask for iDRAC network.
+idrac_network_netmask:
+
+# VLAN for iDRAC network.
+idrac_network_vlan:
+
+# ID of VLAN to use for bootstrapping iDRACs.
+idrac_bootstrap_vlan:
+
+# Name of network namespace on controller to use for bootstrapping iDRACs.
+idrac_bootstrap_net_namespace: idrac-bootstrap
+
+# Base network interface on controller to use for bootstrapping iDRACs.
+idrac_bootstrap_controller_interface:
+
+# VLAN network interface on controller to create for bootstrapping iDRACs.
+idrac_bootstrap_controller_vlan_interface: "{{ idrac_bootstrap_controller_interface }}.{{ idrac_bootstrap_vlan }}"
+
+# IP address of controller to use for bootstrapping iDRACs.
+idrac_bootstrap_controller_ip: 192.168.0.1
+
+# Name of an Ansible group containing switches forming the iDRAC network.
+idrac_bootstrap_switch_group:

ansible/idrac-bootstrap-one.yml

@@ -0,0 +1,190 @@
+---
+# This is a tasks file used by the idrac-bootstrap.yml playbook to support
+# bootstrapping the network configuration of a single iDRAC.
+# We use the following procedure to configure the iDRAC:
+# 1. Check whether the required IP is already reachable. Skip remaining tasks
+#    if so.
+# 2. Configure the switch interface to which the iDRAC is attached as an
+#    access port on the bootstrap VLAN.
+# 3. Clear the ARP cache on the controller in the bootstrap network namespace.
+# 4. Check whether the iDRAC default IP address is reachable.
+# 5. Enable IPMI on the iDRAC.
+# 6. Configure networking for the iDRAC.
+# 7. Configure the switch interface to which the iDRAC is attached as an
+#    access port on the iDRAC management VLAN.
+
+- name: Check whether we can ping the iDRAC's configured IP address
+  command: "ping -c 1 {{ idrac_network_ip }}"
+  run_once: True
+  # We use this convoluted method to allow a delegate_to with a variable host.
+  # See http://www.elmund.io/configuration%20management/2015/07/23/ansible-delegate_to-and-variables/.
+  with_items:
+    - "{{ idrac_bootstrap_controller }}"
+  loop_control:
+    loop_var: delegate_host
+  delegate_to: "{{ delegate_host }}"
+  register: ping_result
+  changed_when: False
+  failed_when: False
+
+- name: Set a fact about whether the iDRAC requires bootstrapping
+  set_fact:
+    idrac_bootstrap_required: "{{ ping_result.results[0].rc != 0 }}"
+  run_once: True
+
+- name: Display the result of the ping
+  debug:
+    msg: >
+      The iDRAC on switch port with description {{ idrac_port_description }}
+      and configured IP address {{ idrac_network_ip }} was
+      {{ 'un' if idrac_bootstrap_required else '' }}reachable. The iDRAC will
+      {{ '' if idrac_bootstrap_required else 'not ' }}be bootstrapped.
+  run_once: True
+
+# The tasks in this block are only executed when the bootstrap is required.
+- block:
+    - name: Ensure DellOS6 switch interface is a member of the bootstrap VLAN
+      dellos6_config:
+        provider: "{{ dell_switch_provider }}"
+        lines:
+          - "switchport access vlan {{ idrac_bootstrap_vlan }}"
+        parents:
+          - "interface {{ switch_interface_name }}"
+      delegate_to: localhost
+      when: "{{ switch_type == 'dellos6' }}"
+
+    # The tasks in this block are delegated to the controller.
+    - block:
+        - name: Ensure the iDRAC default IP address is removed from the controller's ARP cache
+          command: >
+            ip netns exec {{ idrac_bootstrap_net_namespace }}
+            arp -d {{ idrac_default_ip }}
+          become: True
+          with_items:
+            - "{{ idrac_bootstrap_controller }}"
+          loop_control:
+            loop_var: delegate_host
+          delegate_to: "{{ delegate_host }}"
+          register: arp_result
+          failed_when:
+            - "{{ arp_result | failed }}"
+            - "{{ 'No ARP entry for ' ~ idrac_default_ip not in arp_result.stdout }}"
+
+        # Ansible's until keyword seems to not work nicely with failed_when, causing
+        # the task to fail even though we have specified failed_when: False.
+        - name: Check whether we can ping the iDRAC's default IP address
+          shell: |
+            max_attempts=3
+            interval=5
+            for attempt in $(seq $max_attempts); do
+              ip netns exec {{ idrac_bootstrap_net_namespace }} \
+              ping -c 1 {{ idrac_default_ip }}
+              ping_rc=$?
+              if [[ $ping_rc -eq 0 ]] || [[ $attempt -eq $max_attempts ]]; then
+                break
+              fi
+              sleep $interval
+            done
+            exit $ping_rc
+          become: True
+          with_items:
+            - "{{ idrac_bootstrap_controller }}"
+          loop_control:
+            loop_var: delegate_host
+          delegate_to: "{{ delegate_host }}"
+          register: ping_result
+          changed_when: False
+          failed_when: False
+
+        - name: Initialise a fact about whether iDRAC bootstrap failed
+          set_fact:
+            idrac_bootstrap_failure: {}
+
+        - name: Set a fact about whether the iDRAC default IP was reachable
+          set_fact:
+            idrac_bootstrap_failure: "{{ ping_result.results[0] }}"
+          when: "{{ ping_result.results[0].rc != 0 }}"
+
+        - name: Ensure IPMI is enabled on the iDRAC
+          command: >
+            ip netns exec {{ idrac_bootstrap_net_namespace }}
+            /opt/dell/srvadmin/bin/idracadm7
+            -r {{ idrac_default_ip }} -u {{ idrac_default_username }} -p {{ idrac_default_password }}
+            set iDRAC.IPMILan.Enable 1
+          become: True
+          with_items:
+            - "{{ idrac_bootstrap_controller }}"
+          loop_control:
+            loop_var: delegate_host
+          delegate_to: "{{ delegate_host }}"
+          when: "{{ not idrac_bootstrap_failure }}"
+          register: racadm_ipmi_enable
+          failed_when: False
+
+        - name: Set a fact about whether enabling IPMI on the iDRAC failed
+          set_fact:
+            idrac_bootstrap_failure: "{{ racadm_ipmi_enable.results[0] }}"
+          when:
+            - "{{ not idrac_bootstrap_failure }}"
+            - "{{ racadm_ipmi_enable.results[0].rc != 0 }}"
+
+        - name: Ensure the iDRAC IP address is configured
+          command: >
+            ip netns exec {{ idrac_bootstrap_net_namespace }}
+            /opt/dell/srvadmin/bin/idracadm7
+            -r {{ idrac_default_ip }} -u {{ idrac_default_username }} -p {{ idrac_default_password }}
+            setniccfg -s {{ idrac_network_ip }} {{ idrac_network_netmask }} {{ idrac_network_gateway }}
+          become: True
+          with_items:
+            - "{{ idrac_bootstrap_controller }}"
+          loop_control:
+            loop_var: delegate_host
+          delegate_to: "{{ delegate_host }}"
+          when:
+            - "{{ not idrac_bootstrap_failure }}"
+          register: racadm_setniccfg
+          failed_when: False
+
+        - name: Set a fact about whether setting network configuration on the iDRAC failed
+          set_fact:
+            idrac_bootstrap_failure: "{{ racadm_setniccfg.results[0] }}"
+          when:
+            - "{{ not idrac_bootstrap_failure }}"
+            - "{{ racadm_setniccfg.results[0].rc != 0 }}"
+
+        - name: Append the iDRAC to the successful list on success
+          set_fact:
+            idrac_bootstrap_success: >
+              {{ idrac_bootstrap_success + [idrac_port_description] }}
+          when: "{{ not idrac_bootstrap_failure }}"
+
+        - name: Append the iDRAC to the failed list on failure
+          set_fact:
+            idrac_bootstrap_failed: >
+              {{ idrac_bootstrap_failed +
+                 [{"port description": idrac_port_description,
+                   "failure": idrac_bootstrap_failure}] }}
+          when: "{{ idrac_bootstrap_failure }}"
+      run_once: True
+
+  # Ensure we reconfigure the switch interface.
+  always:
+    - name: Ensure DellOS6 switch iDRAC interface is a member of the management VLAN
+      dellos6_config:
+        provider: "{{ dell_switch_provider }}"
+        lines:
+          - "switchport access vlan {{ idrac_network_vlan }}"
+        parents:
+          - "interface {{ switch_interface_name }}"
+      delegate_to: localhost
+      when:
+        - "{{ switch_type == 'dellos6' }}"
+
+  when: "{{ idrac_bootstrap_required }}"
+
+- name: Append the iDRAC to the unchanged list when unchanged
+  set_fact:
+    idrac_bootstrap_unchanged: >
+      {{ idrac_bootstrap_unchanged + [idrac_port_description] }}
+  run_once: True
+  when: "{{ not idrac_bootstrap_required }}"

ansible/idrac-bootstrap.yml

@@ -0,0 +1,219 @@
+---
+# This playbook is intended to bootstrap network configuration of the iDRACs on
+# a set of Dell servers. By default, iDRACs have a known static IP address
+# configured. Since all iDRACs have the same default IP address, we need a way
+# to isolate a single iDRAC while we set its network configuration. We do this
+# using a temporary VLAN accessible from one of the controllers.
+#
+# We use the following procedure:
+# 1. Create a VLAN interface on the controller node with IP in the iDRAC
+#    default subnet.
+# 2. Create the temporary bootstrap VLAN on the switch, accessible by the
+#    controller and trunked to all switches within the network.
+# 3. For each iDRAC switch port in turn, flip to the temporary VLAN and
+#    configure the iDRAC's IP address, before returning the port to the iDRAC
+#    management VLAN.
+# 4. Remove the temporary bootstrap VLAN from the switch.
+# 5. Remove the VLAN interface on the controller node.
+
+- name: Ensure the iDRAC switches are supported
+  hosts: "{{ idrac_bootstrap_switch_group }}"
+  gather_facts: no
+  vars:
+    supported_switch_types:
+      - dellos6
+  tasks:
+    - name: Ensure switch type is supported
+      fail:
+        msg: >
+          The iDRAC bootstrap process currently only supports DellOS6 switches.
+      when: "{{ switch_type not in supported_switch_types }}"
+
+# 1. Create a VLAN interface on the controller node with IP in the iDRAC
+#    default subnet.
+- name: Ensure the controller bootstrap network is configured
+  hosts: controllers[0]
+  tasks:
+    # Install Dell server admin tools.
+    - block:
+        - name: Ensure wget is installed
+          yum:
+            name: wget
+            state: installed
+
+        - name: Ensure Dell srvadmin repository is installed
+          shell: "wget -q -O - http://linux.dell.com/repo/hardware/latest/bootstrap.cgi | bash"
+
+        - name: Ensure Dell srvadmin-idrac7 package is installed
+          yum:
+            name: srvadmin-idrac7
+            state: installed
+
+        # Configure access to the temporary network on a controller.
+        - block:
+            # Clear any previous state.
+            - name: Ensure iDRAC bootstrap network namespace is deleted from the controller
+              command: "ip netns delete {{ idrac_bootstrap_net_namespace }}"
+              args:
+                removes: "/var/run/netns/{{ idrac_bootstrap_net_namespace }}"
+
+            - name: Ensure iDRAC bootstrap network namespace exists on controller
+              command: "ip netns add {{ idrac_bootstrap_net_namespace }}"
+
+            - name: Ensure bootstrap VLAN interface exists on the controller
+              command: "ip link add link {{ idrac_bootstrap_controller_interface }} name {{ idrac_bootstrap_controller_vlan_interface }} type vlan id {{ idrac_bootstrap_vlan }}"
+
+            - name: Ensure bootstrap VLAN interface is in network namespace
+              command: "ip link set {{ idrac_bootstrap_controller_vlan_interface }} netns {{ idrac_bootstrap_net_namespace }}"
+
+            - name: Ensure the bootstrap VLAN interface is active
+              command: "ip netns exec {{ idrac_bootstrap_net_namespace }} ip link set {{ idrac_bootstrap_controller_vlan_interface }} up"
+
+            - name: Ensure the bootstrap VLAN interface IP address is configured
+              command: "ip netns exec {{ idrac_bootstrap_net_namespace }} ip address add {{ idrac_bootstrap_controller_ip }}/24 dev {{ idrac_bootstrap_controller_vlan_interface }}"
+          rescue:
+            - name: Rescue | Ensure the bootstrap network namespace is removed from the controller
+              command: "ip netns delete {{ idrac_bootstrap_net_namespace }}"
+
+            - name: Rescue | Fail playbook execution on error
+              fail:
+                msg: >
+                  Failed to configure access to temporary iDRAC bootstrap
+                  network on controller.
+      become: True
+
+# 2. Create the temporary bootstrap VLAN on the switch, accessible by the
+#    controller and trunked to all switches within the network.
+- name: Ensure the bootstrap VLAN is configured on switches
+  hosts: "{{ idrac_bootstrap_switch_group }}"
+  gather_facts: no
+  vars:
+    switch_interface_config_bootstrap_trunk:
+      config:
+        - "switchport trunk allowed vlan add {{ idrac_bootstrap_vlan }}"
+    # Initialise the switch interface configuration.
+    switch_interface_config_bootstrap: {}
+  pre_tasks:
+    - name: Update facts about switch trunk interfaces
+      set_fact:
+        switch_interface_config_bootstrap: >
+          {{ switch_interface_config_bootstrap | combine({item.key: switch_interface_config_bootstrap_trunk}) }}
+      with_dict: "{{ switch_interface_config }}"
+      when: >
+        {{ item.value.description == groups['controllers'][0] or
+           item.value.description | replace('-trunk', '') in groups[idrac_bootstrap_switch_group] }}
+
+  roles:
+    # Configure bootstrap VLAN on the switch and add controller and trunk
+    # interfaces to it.
+    - role: dell-switch
+      dell_switch_delegate_to: localhost
+      dell_switch_type: "{{ switch_type }}"
+      dell_switch_provider: "{{ switch_dellos_provider }}"
+      dell_switch_config:
+        - "vlan {{ idrac_bootstrap_vlan }}"
+      dell_switch_interface_config: "{{ switch_interface_config_bootstrap }}"
+      when: "{{ switch_interface_config_bootstrap != {} }}"
+
+# 3. For each iDRAC switch port in turn, flip to the temporary VLAN and
+#    configure the iDRAC's IP address, before returning the port to the iDRAC
+#    management VLAN.
+- name: Ensure iDRACs are bootstrapped
+  hosts: "{{ idrac_bootstrap_switch_group }}"
+  gather_facts: no
+  # This is a separate play so that we can apply the serial keyword.
+  serial: 1
+  tasks:
+    - name: Initialise facts containing successful, unchanged and failed iDRACs
+      set_fact:
+        idrac_bootstrap_success: []
+        idrac_bootstrap_unchanged: []
+        idrac_bootstrap_failed: []
+
+    # Iterate over each switch port with an iDRAC attached in turn.
+    - name: Ensure iDRACs are (sequentially) bootstrapped
+      include: idrac-bootstrap-one.yml
+      vars:
+        dell_switch_delegate_to: localhost
+        dell_switch_type: "{{ switch_type }}"
+        dell_switch_provider: "{{ switch_dellos_provider }}"
+        switch_interface_name: "{{ item.key }}"
+        idrac_port_description: "{{ item.value.description }}"
+        idrac_network_ip: "{{ idrac_network_ips[idrac_port_description] }}"
+        idrac_bootstrap_controller: "{{ hostvars[groups['controllers'][0]].ansible_host }}"
+      with_dict: "{{ switch_interface_config }}"
+      when: "{{ item.value.description in idrac_network_ips }}"
+
+# 4. Remove the temporary bootstrap VLAN from the switch.
+- name: Ensure the bootstrap VLAN is removed from switches
+  hosts: "{{ idrac_bootstrap_switch_group }}"
+  gather_facts: no
+  vars:
+    switch_interface_config_bootstrap_trunk:
+      config:
+        - "switchport trunk allowed vlan remove {{ idrac_bootstrap_vlan }}"
+    # Initialise the switch interface configuration.
+    switch_interface_config_bootstrap: {}
+  pre_tasks:
+    - name: Update facts about switch trunk interfaces
+      set_fact:
+        switch_interface_config_bootstrap: >
+          {{ switch_interface_config_bootstrap | combine({item.key: switch_interface_config_bootstrap_trunk}) }}
+      with_dict: "{{ switch_interface_config }}"
+      when: >
+        {{ item.value.description == groups['controllers'][0] or
+           item.value.description | replace('-trunk', '') in groups[idrac_bootstrap_switch_group] }}
+  roles:
+    # Remove bootstrap VLAN from the switch and remove controller and trunk
+    # interfaces from it.
+    - role: dell-switch
+      dell_switch_delegate_to: localhost
+      dell_switch_type: "{{ switch_type }}"
+      dell_switch_provider: "{{ switch_dellos_provider }}"
+      dell_switch_config:
+        - "no vlan {{ idrac_bootstrap_vlan }}"
+      dell_switch_interface_config: "{{ switch_interface_config_bootstrap }}"
+      when: "{{ switch_interface_config_bootstrap != {} }}"
+
+# 5. Remove the VLAN interface on the controller node.
+- name: Ensure the controller bootstrap network is cleaned up
+  hosts: controllers[0]
+  tasks:
+    # This should also delete the network interface within the namespace.
+    - name: Ensure the bootstrap network namespace is removed from the controller
+      command: "ip netns delete {{ idrac_bootstrap_net_namespace }}"
+      become: True
+
+- name: Display the results of the iDRAC bootstrap procedure
+  hosts: "{{ idrac_bootstrap_switch_group }}"
+  gather_facts: no
+  tasks:
+    - name: Display a list of failed iDRACs
+      set_fact:
+        idrac_bootstrap_failed_port_descriptions: "{{ idrac_bootstrap_failed | map(attribute='port description') | list }}"
+      when: "{{ idrac_bootstrap_failed | length > 0 }}"
+
+    - name: Display a list of successfully bootstrapped iDRACs
+      debug:
+        var: idrac_bootstrap_success
+
+    - name: Display a list of iDRACs that did not require bootstrapping
+      debug:
+        var: idrac_bootstrap_unchanged
+
+    - name: Display a list of failed iDRACs
+      debug:
+        var: idrac_bootstrap_failed_port_descriptions
+      when: "{{ idrac_bootstrap_failed | length > 0 }}"
+
+    - name: Display a list of failed iDRACs with debug output for the failed tasks
+      debug:
+        var: idrac_bootstrap_failed
+      when: "{{ idrac_bootstrap_failed | length > 0 }}"
+
+    - name: Fail if there were any iDRAC bootstrapping failures
+      fail:
+        msg: >
+          One or more iDRACs failed to bootstrap, see the list above for
+          details.
+      when: "{{ idrac_bootstrap_failed | length > 0 }}"

etc/kayobe/idrac.yml

@@ -0,0 +1,43 @@
+---
+###############################################################################
+# iDRAC configuration.
+
+# Default username for iDRACs.
+#idrac_default_username:
+
+# Default password for iDRACs.
+#idrac_default_password:
+
+# Default IP address for iDRACs.
+#idrac_default_ip:
+
+# Dict mapping host description (as found in switch interface configuration) to
+# IP address of the iDRAC for that host.
+#idrac_network_ips:
+
+# Gateway IP address for iDRAC network.
+#idrac_network_gateway:
+
+# IP netmask for iDRAC network.
+#idrac_network_netmask:
+
+# VLAN for iDRAC network.
+#idrac_network_vlan:
+
+# ID of VLAN to use for bootstrapping iDRACs.
+#idrac_bootstrap_vlan:
+
+# Name of network namespace on controller to use for bootstrapping iDRACs.
+#idrac_bootstrap_net_namespace:
+
+# Base network interface on controller to use for bootstrapping iDRACs.
+#idrac_bootstrap_controller_interface:
+
+# VLAN network interface on controller to create for bootstrapping iDRACs.
+#idrac_bootstrap_controller_vlan_interface:
+
+# IP address of controller to use for bootstrapping iDRACs.
+#idrac_bootstrap_controller_ip:
+
+# Name of an Ansible group containing switches forming the iDRAC network.
+#idrac_bootstrap_switch_group: