From a2cbe94b7621ce41312bd9a998a3bc4f3ea5afda Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 8 Dec 2017 15:38:54 +0000 Subject: [PATCH] Improve bare metal compute node management Adds these new commands: kayobe baremetal compute inspect kayobe baremetal compute manage kayobe baremetal compute provide These can be used to set the provision state of ironic nodes in the baremetal-compute group. --- .gitignore | 1 + ansible/baremetal-compute-inspect.yml | 49 ++++++++++ ansible/baremetal-compute-manage.yml | 44 +++++++++ ansible/baremetal-compute-provide.yml | 44 +++++++++ ansible/compute-node-provide.yml | 89 +------------------ .../activate-virtualenv/defaults/main.yml | 3 + .../roles/activate-virtualenv/tasks/main.yml | 10 +++ .../deactivate-virtualenv/defaults/main.yml | 5 ++ .../deactivate-virtualenv/tasks/main.yml | 6 ++ doc/source/administration.rst | 26 ++++++ doc/source/release-notes.rst | 3 + kayobe/cli/commands.py | 28 ++++++ kayobe/tests/unit/cli/test_commands.py | 54 +++++++++++ requirements.yml | 1 + setup.py | 3 + 15 files changed, 278 insertions(+), 88 deletions(-) create mode 100644 ansible/baremetal-compute-inspect.yml create mode 100644 ansible/baremetal-compute-manage.yml create mode 100644 ansible/baremetal-compute-provide.yml mode change 100644 => 120000 ansible/compute-node-provide.yml create mode 100644 ansible/roles/activate-virtualenv/defaults/main.yml create mode 100644 ansible/roles/activate-virtualenv/tasks/main.yml create mode 100644 ansible/roles/deactivate-virtualenv/defaults/main.yml create mode 100644 ansible/roles/deactivate-virtualenv/tasks/main.yml diff --git a/.gitignore b/.gitignore index 14039fc2c..0074ab611 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ ansible/roles/stackhpc.drac/ ansible/roles/stackhpc.drac-facts/ ansible/roles/stackhpc.os-flavors/ ansible/roles/stackhpc.os-images/ +ansible/roles/stackhpc.os-ironic-state/ ansible/roles/stackhpc.os-openstackclient/ ansible/roles/stackhpc.os-networks/ ansible/roles/stackhpc.os-projects/ diff --git a/ansible/baremetal-compute-inspect.yml b/ansible/baremetal-compute-inspect.yml new file mode 100644 index 000000000..f2a499020 --- /dev/null +++ b/ansible/baremetal-compute-inspect.yml @@ -0,0 +1,49 @@ +--- +# This playbook will ensure that all baremetal compute nodes in the +# baremetal-compute ansible group are inspected. The nodes should be in the +# 'manageable' state. + +# We install shade in a virtualenv on one of the controllers, and delegate to +# it when executing the stackhpc.os-ironic-state role. + +- name: Ensure dependencies are installed and the virtual environment is activated + hosts: controllers[0] + gather_facts: False + vars: + venv: "{{ virtualenv_path }}/shade" + roles: + - role: stackhpc.os-shade + os_shade_venv: "{{ venv }}" + + - role: activate-virtualenv + activate_virtualenv_path: "{{ venv }}" + +- name: Ensure baremetal compute nodes are inspected in ironic + hosts: baremetal-compute + gather_facts: False + vars: + controller_host: "{{ groups['controllers'][0] }}" + # Whether to wait for the state transition to complete. + baremetal_compute_wait: True + # Time to wait for state transition to complete, if baremetal_compute_wait + # is True. + baremetal_compute_timeout: 1200 + tasks: + - name: Ensure baremetal compute nodes are inspected in ironic + os_ironic_inspect: + auth_type: "{{ openstack_auth_type }}" + auth: "{{ openstack_auth }}" + name: "{{ inventory_hostname }}" + timeout: "{{ baremetal_compute_timeout }}" + wait: "{{ baremetal_compute_wait }}" + delegate_to: "{{ controller_host }}" + vars: + # NOTE: Without this, the controller's ansible_host variable will not + # be respected when using delegate_to. + ansible_host: "{{ hostvars[controller_host].ansible_host | default(controller_host) }}" + +- name: Deactivate the virtual environment on the controller + hosts: controllers[0] + gather_facts: False + roles: + - role: deactivate-virtualenv diff --git a/ansible/baremetal-compute-manage.yml b/ansible/baremetal-compute-manage.yml new file mode 100644 index 000000000..a017ba4e3 --- /dev/null +++ b/ansible/baremetal-compute-manage.yml @@ -0,0 +1,44 @@ +--- +# This playbook will ensure that all baremetal compute nodes in the overcloud +# ironic inventory are manageable. Supported initial states include 'enroll', +# 'manageable', and 'available'. + +# We install shade in a virtualenv on one of the controllers, and delegate to +# it when executing the stackhpc.os-ironic-state role. + +- name: Ensure baremetal compute nodes are available in ironic + hosts: controllers[0] + gather_facts: False + vars: + venv: "{{ virtualenv_path }}/shade" + roles: + - role: stackhpc.os-shade + os_shade_venv: "{{ venv }}" + + - role: activate-virtualenv + activate_virtualenv_path: "{{ venv }}" + +- name: Ensure baremetal compute nodes are manageable in ironic + hosts: baremetal-compute + gather_facts: False + vars: + # Whether to wait for the state transition to complete. + baremetal_compute_wait: True + # Time to wait for state transition to complete, if baremetal_compute_wait + # is True. + baremetal_compute_timeout: 1200 + roles: + - role: stackhpc.os-ironic-state + os_ironic_state_auth_type: "{{ openstack_auth_type }}" + os_ironic_state_auth: "{{ openstack_auth }}" + os_ironic_state_name: "{{ inventory_hostname }}" + os_ironic_state_provision_state: "manage" + os_ironic_state_wait: "{{ baremetal_compute_wait }}" + os_ironic_state_timeout: "{{ baremetal_compute_timeout }}" + os_ironic_state_delegate_to: "{{ groups['controllers'][0] }}" + +- name: Ensure baremetal compute nodes are available in ironic + hosts: controllers[0] + gather_facts: False + roles: + - role: deactivate-virtualenv diff --git a/ansible/baremetal-compute-provide.yml b/ansible/baremetal-compute-provide.yml new file mode 100644 index 000000000..6ada8c666 --- /dev/null +++ b/ansible/baremetal-compute-provide.yml @@ -0,0 +1,44 @@ +--- +# This playbook will ensure that all baremetal compute nodes in the overcloud +# ironic inventory are available. Supported initial states include 'enroll' and +# 'manageable'. + +# We install shade in a virtualenv on one of the controllers, and delegate to +# it when executing the stackhpc.os-ironic-state role. + +- name: Ensure baremetal compute nodes are available in ironic + hosts: controllers[0] + gather_facts: False + vars: + venv: "{{ virtualenv_path }}/shade" + roles: + - role: stackhpc.os-shade + os_shade_venv: "{{ venv }}" + + - role: activate-virtualenv + activate_virtualenv_path: "{{ venv }}" + +- name: Ensure baremetal compute nodes are available in ironic + hosts: baremetal-compute + gather_facts: False + vars: + # Whether to wait for the state transition to complete. + baremetal_compute_wait: True + # Time to wait for state transition to complete, if baremetal_compute_wait + # is True. + baremetal_compute_timeout: 1200 + roles: + - role: stackhpc.os-ironic-state + os_ironic_state_auth_type: "{{ openstack_auth_type }}" + os_ironic_state_auth: "{{ openstack_auth }}" + os_ironic_state_name: "{{ inventory_hostname }}" + os_ironic_state_provision_state: "provide" + os_ironic_state_wait: "{{ baremetal_compute_wait }}" + os_ironic_state_timeout: "{{ baremetal_compute_timeout }}" + os_ironic_state_delegate_to: "{{ groups['controllers'][0] }}" + +- name: Ensure baremetal compute nodes are available in ironic + hosts: controllers[0] + gather_facts: False + roles: + - role: deactivate-virtualenv diff --git a/ansible/compute-node-provide.yml b/ansible/compute-node-provide.yml deleted file mode 100644 index 3c0786613..000000000 --- a/ansible/compute-node-provide.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- -# This playbook will ensure that all baremetal compute nodes in the overcloud -# ironic inventory are available. Supported initial states include 'enroll' and -# 'manageable'. - -- name: Ensure baremetal compute nodes are available in ironic - hosts: controllers[0] - vars: - venv: "{{ virtualenv_path }}/shade" - # Set this to a colon-separated list of baremetal compute node hostnames to - # provide. If unset, all baremetal compute nodes will be provided. - compute_node_limit: "" - compute_node_limit_list: "{{ compute_node_limit.split(':') }}" - roles: - - role: stackhpc.os-openstackclient - os_openstackclient_venv: "{{ venv }}" - tasks: - - name: Ensure required Python packages are installed - pip: - name: "{{ item.name }}" - state: present - virtualenv: "{{ venv }}" - with_items: - - name: python-ironicclient - - - name: Get a list of ironic nodes - shell: > - source {{ venv }}/bin/activate && - openstack baremetal node list --fields name provision_state -f json --no-maintenance - register: ironic_node_list - changed_when: False - environment: "{{ openstack_auth_env }}" - - - name: Initialise a fact containing the ironic nodes - set_fact: - ironic_nodes: [] - - - name: Update a fact containing the ironic nodes - set_fact: - ironic_nodes: "{{ ironic_nodes + [item] }}" - with_items: "{{ ironic_node_list.stdout | from_json }}" - when: > - {{ not compute_node_limit or - item['Name'] in compute_node_limit_list }} - - - name: Ensure ironic nodes are managed - shell: > - source {{ venv }}/bin/activate && - openstack baremetal node manage {{ item['Name'] }} - with_items: "{{ ironic_nodes }}" - when: item['Provisioning State'] == 'enroll' - environment: "{{ openstack_auth_env }}" - - - name: Ensure ironic nodes are available - shell: > - source {{ venv }}/bin/activate && - openstack baremetal node provide {{ item['Name'] }} - with_items: "{{ ironic_nodes }}" - when: item['Provisioning State'] in ['enroll', 'manageable'] - environment: "{{ openstack_auth_env }}" - - - name: Get a list of ironic nodes - shell: > - source {{ venv }}/bin/activate && - openstack baremetal node list -f json -c Name -c 'Provisioning State' --no-maintenance - register: ironic_node_list - changed_when: False - environment: "{{ openstack_auth_env }}" - - - name: Initialise a fact containing the ironic nodes - set_fact: - ironic_nodes: [] - - - name: Limit ironic nodes to the specified list - set_fact: - ironic_nodes: "{{ ironic_nodes + [item] }}" - with_items: "{{ ironic_node_list.stdout | from_json }}" - when: > - {{ not compute_node_limit or - item['Name'] in compute_node_limit_list }} - - - name: Fail if any ironic nodes are not available - fail: - msg: > - Failed to make baremetal compute node {{ item['Name'] }} available in ironic. - Provisioning state is {{ item['Provisioning State'] }}. - with_items: "{{ ironic_nodes }}" - when: item['Provisioning State'] != 'available' diff --git a/ansible/compute-node-provide.yml b/ansible/compute-node-provide.yml new file mode 120000 index 000000000..1edf8f78b --- /dev/null +++ b/ansible/compute-node-provide.yml @@ -0,0 +1 @@ +baremetal-compute-provide.yml \ No newline at end of file diff --git a/ansible/roles/activate-virtualenv/defaults/main.yml b/ansible/roles/activate-virtualenv/defaults/main.yml new file mode 100644 index 000000000..88ddb6608 --- /dev/null +++ b/ansible/roles/activate-virtualenv/defaults/main.yml @@ -0,0 +1,3 @@ +--- +# Path to a virtualenv to activate. +activate_virtualenv_path: diff --git a/ansible/roles/activate-virtualenv/tasks/main.yml b/ansible/roles/activate-virtualenv/tasks/main.yml new file mode 100644 index 000000000..fce768d6d --- /dev/null +++ b/ansible/roles/activate-virtualenv/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Set a fact containing the current python interpreter + set_fact: + activate_virtualenv_current_python_interpreter: "{{ ansible_python_interpreter | default('/usr/bin/python') }}" + +# Note that setting this via a play or task variable seems to not +# evaluate the Jinja variable reference, so we use set_fact. +- name: Update the Ansible python interpreter fact to point to the virtualenv + set_fact: + ansible_python_interpreter: "{{ activate_virtualenv_path }}/bin/python" diff --git a/ansible/roles/deactivate-virtualenv/defaults/main.yml b/ansible/roles/deactivate-virtualenv/defaults/main.yml new file mode 100644 index 000000000..726f74f2c --- /dev/null +++ b/ansible/roles/deactivate-virtualenv/defaults/main.yml @@ -0,0 +1,5 @@ +--- +# Path to a python interpreter to set as the ansible_python_interpreter +# variable. The default uses a variable set by the activate-virtualenv role +# containing the original python interpreter before entering the virtualenv. +deactivate_virtualenv_python_interpreter: "{{ activate_virtualenv_current_python_interpreter | default('/usr/bin/python') }}" diff --git a/ansible/roles/deactivate-virtualenv/tasks/main.yml b/ansible/roles/deactivate-virtualenv/tasks/main.yml new file mode 100644 index 000000000..36fc4d9c2 --- /dev/null +++ b/ansible/roles/deactivate-virtualenv/tasks/main.yml @@ -0,0 +1,6 @@ +--- +# This variable is unset before we set it, and it does not appear to be +# possible to unset a variable in Ansible. +- name: Set a fact to reset the Ansible python interpreter + set_fact: + ansible_python_interpreter: "{{ deactivate_virtualenv_python_interpreter }}" diff --git a/doc/source/administration.rst b/doc/source/administration.rst index 406f8dd90..51ad20d27 100644 --- a/doc/source/administration.rst +++ b/doc/source/administration.rst @@ -125,6 +125,32 @@ any of these hosts are not expected to be active (e.g. prior to overcloud deployment), the set of target hosts may be limited using the ``--limit`` argument. +Baremetal Compute Node Management +================================= + +When enrolling new hardware or performing maintenance, it can be useful to be +able to manage many bare metal compute nodes simulteneously. + +In all cases, commands are delegated to one of the controller hosts, and +executed concurrently. Note that ansible's ``forks`` configuration option, +which defaults to 5, may limit the number of nodes configured concurrently. + +To move the baremetal compute nodes to the ``manageable`` provision state:: + + (kayobe) $ kayobe baremetal compute manage + +To move the baremetal compute nodes to the ``available`` provision state:: + + (kayobe) $ kayobe baremetal compute provide + +To trigger hardware inspection on the baremetal compute nodes:: + + (kayobe) $ kayobe baremetal compute inspect + +By default these commands wait for the state transition to complete for each +node. This behavior can be changed by overriding the variable +``baremetal_compute_wait`` via ``-e baremetal_compute_wait=False`` + Running Kayobe Playbooks on Demand ================================== diff --git a/doc/source/release-notes.rst b/doc/source/release-notes.rst index 2068213af..9e7aa545a 100644 --- a/doc/source/release-notes.rst +++ b/doc/source/release-notes.rst @@ -28,6 +28,9 @@ Features be added to the ``[compute]`` group. * Adds support for multiple external networks. ``external_net_names`` should be a list of names of networks. +* Adds commands for management of baremetal compute nodes - ``kayobe baremetal + compute inspect``, ``kayobe baremetal compute manage``, and ``kayobe + baremetal compute provide``. Upgrade Notes ------------- diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 3552c1833..0d92af956 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -997,3 +997,31 @@ class NetworkConnectivityCheck(KayobeAnsibleMixin, VaultMixin, Command): self.app.LOG.debug("Performing network connectivity check") playbooks = _build_playbook_list("network-connectivity") self.run_kayobe_playbooks(parsed_args, playbooks) + + +class BaremetalComputeInspect(KayobeAnsibleMixin, VaultMixin, Command): + """Perform hardware inspection on baremetal compute nodes.""" + + def take_action(self, parsed_args): + self.app.LOG.debug("Performing hardware inspection on baremetal " + "compute nodes") + playbooks = _build_playbook_list("baremetal-compute-inspect") + self.run_kayobe_playbooks(parsed_args, playbooks) + + +class BaremetalComputeManage(KayobeAnsibleMixin, VaultMixin, Command): + """Put baremetal compute nodes into the manageable provision state.""" + + def take_action(self, parsed_args): + self.app.LOG.debug("Making baremetal compute nodes manageable") + playbooks = _build_playbook_list("baremetal-compute-manage") + self.run_kayobe_playbooks(parsed_args, playbooks) + + +class BaremetalComputeProvide(KayobeAnsibleMixin, VaultMixin, Command): + """Put baremetal compute nodes into the available provision state.""" + + def take_action(self, parsed_args): + self.app.LOG.debug("Making baremetal compute nodes available") + playbooks = _build_playbook_list("baremetal-compute-provide") + self.run_kayobe_playbooks(parsed_args, playbooks) diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 8baf31153..c6ba5b685 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -179,3 +179,57 @@ class TestCase(unittest.TestCase): ), ] self.assertEqual(expected_calls, mock_run.call_args_list) + + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + def test_baremetal_compute_inspect(self, mock_run): + command = commands.BaremetalComputeInspect(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + result = command.run(parsed_args) + self.assertEqual(0, result) + expected_calls = [ + mock.call( + mock.ANY, + [ + "ansible/baremetal-compute-inspect.yml", + ], + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + def test_baremetal_compute_manage(self, mock_run): + command = commands.BaremetalComputeManage(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + result = command.run(parsed_args) + self.assertEqual(0, result) + expected_calls = [ + mock.call( + mock.ANY, + [ + "ansible/baremetal-compute-manage.yml", + ], + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) + + @mock.patch.object(commands.KayobeAnsibleMixin, + "run_kayobe_playbooks") + def test_baremetal_compute_provide(self, mock_run): + command = commands.BaremetalComputeProvide(TestApp(), []) + parser = command.get_parser("test") + parsed_args = parser.parse_args([]) + result = command.run(parsed_args) + self.assertEqual(0, result) + expected_calls = [ + mock.call( + mock.ANY, + [ + "ansible/baremetal-compute-provide.yml", + ], + ), + ] + self.assertEqual(expected_calls, mock_run.call_args_list) diff --git a/requirements.yml b/requirements.yml index b40d04e0a..ce8d2c6c5 100644 --- a/requirements.yml +++ b/requirements.yml @@ -12,6 +12,7 @@ - src: stackhpc.drac-facts - src: stackhpc.os-flavors - src: stackhpc.os-images +- src: stackhpc.os-ironic-state - src: stackhpc.os-networks - src: stackhpc.os-openstackclient - src: stackhpc.os-projects diff --git a/setup.py b/setup.py index 20abb8d38..5cbcebc16 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,9 @@ setup( 'kayobe-vault-password-helper = kayobe.cmd.kayobe_vault_password_helper:main', ], 'kayobe.cli': [ + 'baremetal_compute_inspect = kayobe.cli.commands:BaremetalComputeInspect', + 'baremetal_compute_manage = kayobe.cli.commands:BaremetalComputeManage', + 'baremetal_compute_provide = kayobe.cli.commands:BaremetalComputeProvide', 'control_host_bootstrap = kayobe.cli.commands:ControlHostBootstrap', 'control_host_upgrade = kayobe.cli.commands:ControlHostUpgrade', 'configuration_dump = kayobe.cli.commands:ConfigurationDump',