openstack/tenks
Code Issues Proposed changes

Merge "Wait for resources to become available"

Browse Source
This commit is contained in:
Zuul 2018-11-19 17:56:30 +00:00 committed by Gerrit Code Review
commit 93cc163d53
10 changed files with 1255 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ansible/deploy.yml
View File

@ -28,5 +28,9 @@
import_playbook: flavor_registration.yml
tags: openstack
- name: Wait for resources to become available
import_playbook: resource_wait.yml
tags: openstack, resource
- name: Clean up Tenks state
import_playbook: cleanup_state.yml

46
ansible/resource_wait.yml Normal file
View File

@ -0,0 +1,46 @@
---
- hosts: localhost
tasks:
- name: Check that OpenStack credentials exist in the environment
fail:
msg: >
$OS_USERNAME was not found in the environment. Ensure the OpenStack
credentials exist in your environment, perhaps by sourcing your RC file.
when: not lookup('env', 'OS_USERNAME')
- name: Gather list of OpenStack services
command: >-
{{ virtualenv_path }}/bin/openstack service list -f json
register: service_list_output
changed_when: false
- block:
- name: Set default value for expected resources
set_fact:
tenks_expected_resources: []
- name: Build list of expected resources
# takes the form: [{ resource_class: CUSTOM_TEST_RC, amount: 2, traits: [] }, ]
vars:
resource:
amount: "{{ spec.count | int }}" # this gets converted back to a string
resource_class: "{{ 'CUSTOM_' ~ spec.ironic_config.resource_class | upper | replace('-', '_') }}"
traits: "{{ spec.ironic_config.traits | default([])}}"
set_fact:
tenks_expected_resources: >-
{{ tenks_expected_resources + [resource] }}
loop: "{{ specs }}"
when: "'ironic_config' in spec"
loop_control:
loop_var: spec
- name: Include the wait-for-resources role
include_role:
name: wait-for-resources
vars:
wait_for_resources_required_resources: "{{ tenks_expected_resources }}"
wait_for_resources_venv: "{{ virtualenv_path }}"
wait_for_resources_python_upper_constraints_url: >-
{{ python_upper_constraints_url }}
# Only attempt to wait for resources when the placement service is running
when: service_list_output.stdout | from_json | selectattr('Type', 'equalto', 'placement') | list | length >= 1

28
ansible/roles/wait-for-resources/defaults/main.yml Normal file
View File

@ -0,0 +1,28 @@
---
# A list of dictionaries, each with the following keys:
# * resource_class - the name of the resource class that you want to be
# available
# * traits - a list of traits a provider requires before that provider is
# considered to provide that resource
# * amount - natural number representing the amount of the resource that
# should be available. This will be tested against the total resource count
# minus the amount that is reserved.
wait_for_resources_required_resources:
# Virtualenv containing OpenStack client
wait_for_resources_venv:
# The resources are polled this many times. The delay between each iteration
# is also configurable, but defaults to 10 seconds
wait_for_resources_retry_limit: 15
# The delay between each iteration.
wait_for_resources_delay:
# The URL of the upper constraints file to pass to pip when installing Python
# packages.
wait_for_resources_python_upper_constraints_url:
# Path on the remote machine to copy the python requirements file to
wait_for_resources_python_requirements_path: /tmp/wait-for-python-requirements.txt

4
ansible/roles/wait-for-resources/files/requirements.txt Normal file
View File

@ -0,0 +1,4 @@
# This file contains the Python packages that are needed in the
# what-for-resource virtual env.
osc-placement>=1.3.0 # Apache License 2.0

383
ansible/roles/wait-for-resources/library/wait_for_resources.py Normal file
View File

@ -0,0 +1,383 @@
#!/usr/bin/python
# Copyright (c) 2018 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
DOCUMENTATION = """
module: wait_for_resources
short_description: Waits for a set of resources to become available in
placement
author: Will Szumski(will@stackhpc.com)
options:
- option-name: resources
description: List of dictionaries describing the resources that should
be available. Each dictionary should contain the keys: resource_class,
amount, and optionally, a list of traits that the provider needs to provide
before it is considered to provide the resource.
required: True
type: list
- option-name: venv
description: Path to a virtualenv containing the OpenStack CLI. It is a
requirement for the placement api plugin (osc-placement) to be installed.
required: False
type: string
- option-name: maximum_retries
description: The maximum number of iterations to poll for the resources
required: False
type: int
- option-name: delay
description: Delay between each iteration of the polling loop in seconds
required: False
type: int
requirements:
- python-openstackclient
- osc-placement
"""
EXAMPLES = """
- name: Wait for resources to become available
wait_for_resources:
resources: {'resource_class: CUSTOM_TEST_RC, 'traits': [], 'amount': 2}
delay: 10
maximum_retries: 15
venv: /path/to/venv
"""
RETURN = """
iterations:
description: The Number of iterations required before the resource became
available
returned: success
type: int
sample: 9
"""
# Need to disable PEP8, as it wants all imports at top of file
from ansible.module_utils.basic import AnsibleModule # noqa
from collections import namedtuple # noqa
import six # noqa
import os # noqa
import time # noqa
_RETRY_LIMIT_FAILURE_TEMPLATE = "exceeded retry limit of {max_retries} " \
"whilst waiting for resources to become " \
"available"
Specifier = namedtuple("Specifier", "name traits")
Provider = namedtuple("Provider", "uuid inventory_list traits")
# Store a list of import errors to report to the user.
IMPORT_ERRORS = []
try:
import json
except Exception as e:
IMPORT_ERRORS.append(e)
def meets_criteria(actual, requested):
"""For each resource, determines whether the total satisfies the amount
requested, ignoring any unrequested resource_classes"""
for specifier in requested.keys():
if specifier not in actual:
return False
if actual[specifier] < requested[specifier]:
return False
return True
def get_openstack_binary_path(module):
"""Returns the path to the openstack binary taking into account the
virtualenv that was specified (if available)"""
venv = module.params["venv"]
if venv:
return os.path.join(venv, "bin", "openstack")
# use openstack in PATH
return "openstack"
def get_inventory(module, provider_uuid):
"""
Gets inventory of resources for a give provider UUID
:param module: ansible module
:param provider_uuid: provider to query
:return: list of dictionaries of the form:
[
{
"allocation_ratio": 16.0,
"total": 24,
"reserved": 0,
"resource_class": "VCPU",
"step_size": 1,
"min_unit": 1,
"max_unit": 24
},
]
"""
cmd = "{openstack} resource provider inventory list {uuid} -f json" \
.format(uuid=provider_uuid,
openstack=get_openstack_binary_path(module))
rc, out, err = module.run_command(cmd)
if rc != 0:
msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
module.fail_json(msg=msg)
return json.loads(out)
def get_providers(module):
"""
Gets a list of resource providers
:param module: ansible module
:return: list of dictionaries of the form:
[
{
"generation": 2,
"uuid": "657c4ab0-de82-4def-b7b0-d13ce672bfd0",
"name": "kayobe-will-master"
},
]
"""
cmd = "{openstack} resource provider list -f json".format(
openstack=get_openstack_binary_path(module)
)
rc, out, err = module.run_command(cmd)
if rc != 0:
msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
module.fail_json(msg=msg)
return json.loads(out)
def get_traits(module, provider_uuid):
"""
Gets a list of traits for a resource provider
:param provider_uuid: the uuid of the provider
:param module: ansible module
:return: set of traits of the form:
{
"HW_CPU_X86_SSE2",
}
"""
cmd = "{openstack} resource provider trait list {uuid} " \
"--os-placement-api-version 1.6 -f json " \
.format(uuid=provider_uuid,
openstack=get_openstack_binary_path(module))
rc, out, err = module.run_command(cmd)
if rc != 0:
msg = "{} failed with return code: {}, stderr: {}".format(cmd, rc, err)
module.fail_json(msg=msg)
raw = json.loads(out)
return set([x["name"] for x in raw])
def merge(x, y, f):
""" Merges two dictionaries. If a key appears in both dictionaries, the
common values are merged using the function ``f``"""
# Start with symmetric difference; keys either in A or B, but not both
merged = {k: x.get(k, y.get(k)) for k in six.viewkeys(x) ^ six.viewkeys(y)}
# Update with `f()` applied to the intersection
merged.update(
{k: f(x[k], y[k]) for k in six.viewkeys(x) & six.viewkeys(y)})
return merged
def collect(specifiers, provider):
"""Given a specifier and a provider, gets the amount of resource that is
available for that given provider"""
inventory = {}
# we want to be able to look up items by resource_class
for item in provider.inventory_list:
inventory[item["resource_class"]] = item
result = {}
for specifier in specifiers:
if specifier.traits != provider.traits:
continue
if specifier.name in inventory:
reserved = inventory[specifier.name]["reserved"]
total_available = inventory[specifier.name]["total"]
result[specifier] = total_available - reserved
break
return result
def get_totals(specifiers, providers):
"""Loops over the providers adding up all of the resources that
are available"""
totals = {}
for specifier in specifiers:
# initialise totals so the combine function does not blow up
totals[specifier] = 0
for provider in providers:
current = collect(specifiers, provider)
totals = merge(totals, current, lambda x, y: x + y)
return totals
def are_resources_available(module, specifiers, expected):
"""
Determines whether or not a set of resources are available
:param module: Ansible module
:param specifiers: tuples of the form: (resource_class, traits)
:param expected: dictionary of the form:
{(resource_class, traits): amount }
:return: True if resource available, otherwise False
"""
providers_raw = get_providers(module)
providers = []
for provider in providers_raw:
uuid = provider["uuid"]
traits = get_traits(module, uuid)
inventory = get_inventory(module, uuid)
provider = Provider(
uuid=uuid,
inventory_list=inventory,
traits=traits
)
providers.append(provider)
actual = get_totals(specifiers, providers)
return meets_criteria(actual, expected)
def wait_for_resources(module):
"""
Waits for a set of resources to become available
:param module: Ansible module
:return: the number of attempts needed
"""
max_retries = module.params["maximum_retries"]
delay = module.params["delay"]
resources = module.params["resources"]
expected = {}
specifiers = []
for resource in resources:
# default value for traits
traits = resource["traits"] if resource["traits"] else []
specifier = Specifier(name=resource["resource_class"],
traits=frozenset(traits))
specifiers.append(specifier)
expected[specifier] = resource["amount"]
for i in range(max_retries):
if are_resources_available(module, specifiers, expected):
return i + 1
time.sleep(delay)
fail_msg = _RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=max_retries)
module.fail_json(msg=fail_msg)
def validate_spec(module, resource, field, type_):
check_resource_msg = (
"Please check the dictionaries in your resources list meet the "
"required specification.")
unset_field_msg = (
"One of your resources does not have the field, {field}, set."
)
field_type_msg = (
"The field, {field}, should be type {type}."
)
if field not in resource:
msg = "{} {}".format(unset_field_msg, check_resource_msg)
module.fail_json(msg=msg.format(field=field))
elif not isinstance(resource[field], type_):
msg = "{} {}".format(field_type_msg, check_resource_msg)
module.fail_json(msg=msg.format(field=field, type=type_.__name__))
def validate_resources(module, specs):
resources = module.params["resources"]
for resource in resources:
for spec in specs:
validate_spec(module, resource, spec["field"], type_=spec["type"])
def get_module():
"""Creates and returns an Ansible module"""
module = AnsibleModule(
argument_spec=dict(
resources=dict(required=True, type='list'),
maximum_retries=dict(required=False, type='int', default=15),
delay=dict(required=False, type='int', default=10),
venv=dict(required=False, type='str', default="")
),
supports_check_mode=True,
)
# note(wszumski): amount seems to get converted back to a string
# https://github.com/ansible/ansible/issues/18095
resources = module.params["resources"]
for resource in resources:
if 'amount' not in resource:
continue
amount = resource["amount"]
if isinstance(amount, str):
try:
amount_int = int(amount)
resource["amount"] = amount_int
except ValueError:
# this will get picked up in validate_resources
pass
# Validate resources list
resources_specs = [
{
'field': 'resource_class',
'type': str
},
{
'field': 'traits',
'type': list
},
{
'field': 'amount',
'type': int
},
]
validate_resources(module, resources_specs)
# Fail if there were any exceptions when importing modules.
if IMPORT_ERRORS:
module.fail_json(msg="Import errors: %s" %
", ".join([repr(e) for e in IMPORT_ERRORS]))
return module
def main():
module = get_module()
attempts_needed = 0
# In check mode, only perform validation of parameters
if not module.check_mode:
attempts_needed = wait_for_resources(module)
# This module doesn't really change anything, it just waits for
# something to change externally
result = {
"changed": False,
"iterations": attempts_needed
}
module.exit_json(**result)
if __name__ == '__main__':
main()

38
ansible/roles/wait-for-resources/tasks/main.yml Normal file
View File

@ -0,0 +1,38 @@
---
# Waits for for the resource tracker to be updated with a given resource
# class
- name: Validate that the virtualenv variable is set
fail:
msg: >-
You must set the variable, wait_for_resources_venv, to use this role.
when: wait_for_resources_venv is none
- name: Validate resources
include_tasks: validate.yml
vars:
wait_for_resources_amount: "{{ item.amount }}"
wait_for_resources_resource_class: "{{ item.resource_class }}"
loop: "{{ wait_for_resources_required_resources }}"
- name: Copy over requirements file
copy:
src: requirements.txt
dest: "{{ wait_for_resources_python_requirements_path }}"
- name: Install dependencies in supplied venv
pip:
requirements: "{{ wait_for_resources_python_requirements_path }}"
extra_args: >-
{%- if wait_for_resources_python_upper_constraints_url -%}
-c {{ wait_for_resources_python_upper_constraints_url }}
{%- endif -%}
virtualenv: "{{ wait_for_resources_venv }}"
- name: Call wait_for_resources module
wait_for_resources:
venv: "{{ wait_for_resources_venv }}"
resources: "{{ wait_for_resources_required_resources }}"
delay: "{{ wait_for_resources_delay | default(omit, true) }}"
maximum_retries: "{{ wait_for_resources_retry_limit }}"

17
ansible/roles/wait-for-resources/tasks/validate.yml Normal file
View File

@ -0,0 +1,17 @@
---
- name: Validate that the target resource class is set
fail:
msg: >-
Each item in the dictionary wait_for_resources_required_resources must contain
the key, resource_class.
when: wait_for_resources_class is none
- name: Validate expected resource count
vars:
count: "{{ wait_for_resources_amount | int(default=-1) }}"
fail:
msg: >-
Each item in the dictionary wait_for_resources_required_resources must contain
the key, amount.
when: wait_for_resources_amount is none or count | int < 0

674
tests/test_tenks_wait_for_resources.py Normal file
View File

@ -0,0 +1,674 @@
# Copyright (c) 2018 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import absolute_import
import copy
import imp
import json
import os
import random
from itertools import repeat, chain, cycle
from ansible.module_utils import basic
from tests.utils import ModuleTestCase, set_module_args, AnsibleExitJson, \
AnsibleFailJson
# Python 2/3 compatibility.
try:
from unittest.mock import MagicMock, patch
except ImportError:
from mock import MagicMock, patch # noqa
# Import method lifted from kolla_ansible's test_merge_config.py
PROJECT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
PLUGIN_FILE = os.path.join(PROJECT_DIR,
'ansible/roles/wait-for-resources/library'
'/wait_for_resources.py')
wait_for = imp.load_source('wait_for_resources', PLUGIN_FILE)
meets_criteria = wait_for.meets_criteria
get_providers = wait_for.get_providers
get_inventory = wait_for.get_inventory
get_traits = wait_for.get_traits
merge = wait_for.merge
get_openstack_binary_path = wait_for.get_openstack_binary_path
Specifier = wait_for.Specifier
inventory_list_out = """[
{
"allocation_ratio": 1.0,
"total": 1,
"reserved": 0,
"resource_class": "CUSTOM_TEST_RC",
"step_size": 1,
"min_unit": 1,
"max_unit": 1
}
]""" # noqa
inventory_custom_b_out = """[
{
"allocation_ratio": 1.0,
"total": 1,
"reserved": 0,
"resource_class": "CUSTOM_B",
"step_size": 1,
"min_unit": 1,
"max_unit": 1
}
]""" # noqa
inventory_reserved_out = """[
{
"allocation_ratio": 1.0,
"total": 1,
"reserved": 1,
"resource_class": "CUSTOM_TEST_RC",
"step_size": 1,
"min_unit": 1,
"max_unit": 1
}
]""" # noqa
inventory_list = [{'allocation_ratio': 1.0,
'max_unit': 1,
'min_unit': 1,
'reserved': 0,
'resource_class': 'CUSTOM_TEST_RC',
'step_size': 1,
'total': 1}]
resource_provider_list_out = """[
{
"generation": 2,
"uuid": "657c4ab0-de82-4def-b7b0-d13ce672bfd0",
"name": "kayobe-will-master"
},
{
"generation": 1,
"uuid": "e2e78f98-d3ec-466a-862b-42d7ef5dca7d",
"name": "e2e78f98-d3ec-466a-862b-42d7ef5dca7d"
},
{
"generation": 1,
"uuid": "07072aea-cc2b-4135-a7e2-3a4dd3a9f629",
"name": "07072aea-cc2b-4135-a7e2-3a4dd3a9f629"
}
]""" # noqa
resource_provider_list = [{'generation': 2, 'name': 'kayobe-will-master',
'uuid': '657c4ab0-de82-4def-b7b0-d13ce672bfd0'},
{'generation': 1,
'name': 'e2e78f98-d3ec-466a-862b-42d7ef5dca7d',
'uuid': 'e2e78f98-d3ec-466a-862b-42d7ef5dca7d'},
{'generation': 1,
'name': '07072aea-cc2b-4135-a7e2-3a4dd3a9f629',
'uuid': '07072aea-cc2b-4135-a7e2-3a4dd3a9f629'}]
resource_provider_traits_out = """[
{
"name": "HW_CPU_X86_SSE2"
},
{
"name": "HW_CPU_X86_CLMUL"
},
{
"name": "HW_CPU_X86_SSE"
},
{
"name": "HW_CPU_X86_ABM"
},
{
"name": "HW_CPU_X86_MMX"
},
{
"name": "HW_CPU_X86_AVX2"
},
{
"name": "HW_CPU_X86_SSE41"
},
{
"name": "HW_CPU_X86_SSE42"
},
{
"name": "HW_CPU_X86_AESNI"
},
{
"name": "HW_CPU_X86_AVX"
},
{
"name": "HW_CPU_X86_VMX"
},
{
"name": "HW_CPU_X86_BMI2"
},
{
"name": "HW_CPU_X86_FMA3"
},
{
"name": "HW_CPU_X86_SSSE3"
},
{
"name": "HW_CPU_X86_F16C"
},
{
"name": "HW_CPU_X86_BMI"
}
]""" # noqa
resource_provider_no_traits_out = """[]
"""
resource_provider_traits = {'HW_CPU_X86_SSE2',
'HW_CPU_X86_CLMUL',
'HW_CPU_X86_SSE',
'HW_CPU_X86_ABM',
'HW_CPU_X86_MMX',
'HW_CPU_X86_AVX2',
'HW_CPU_X86_SSE41',
'HW_CPU_X86_SSE42',
'HW_CPU_X86_AESNI',
'HW_CPU_X86_AVX',
'HW_CPU_X86_VMX',
'HW_CPU_X86_BMI2',
'HW_CPU_X86_FMA3',
'HW_CPU_X86_SSSE3',
'HW_CPU_X86_F16C',
'HW_CPU_X86_BMI'}
def traits_to_json(traits):
return json.dumps(
[{'name': x} for x in traits]
)
def traits(*args):
# converts list to frozenset
return frozenset(*args)
resources_expected = {
Specifier('CUSTOM_B', traits()): 2,
Specifier('CUSTOM_A', traits('TRAIT_A')): 3
}
resources_not_enough = {
Specifier('CUSTOM_B', traits()): 1,
Specifier('CUSTOM_A', traits('TRAIT_A')): 2
}
resources_one_ok = {
Specifier('CUSTOM_B', traits()): 2,
Specifier('CUSTOM_A', traits('TRAIT_A')): 2
}
resources_both_ok = {
Specifier('CUSTOM_B', traits()): 2,
Specifier('CUSTOM_A', traits('TRAIT_A')): 3
}
resources_too_many = {
Specifier('CUSTOM_B', traits()): 5,
Specifier('CUSTOM_A', traits('TRAIT_A')): 6
}
resources_extra_keys = {
Specifier('CUSTOM_B', traits()): 2,
Specifier('CUSTOM_A', traits('TRAIT_A')): 3,
Specifier('CUSTOM_C', traits('TRAIT_A')): 3,
}
resources_key_missing = {
'CUSTOM_B': 2
}
dummy_resources = [
{
"resource_class": "CUSTOM_TEST_RC",
"traits": [],
"amount": 3
},
]
# Scenario definitions - we define these outside the test as we want to pass
# them into the 'patch' decorator
# scenario - resource class is incorrect on first iteration
inventory_resource_class_incorrect = chain(
repeat(inventory_custom_b_out, 3),
repeat(inventory_list_out, 3)
)
# scenario - resource class is reserved on first iteration
inventory_resource_class_reserved = chain(
repeat(inventory_reserved_out, 3),
repeat(inventory_list_out, 3)
)
# scenario - same resource class, different traits - success
# Arbitrary subset
resource_provider_traits_subset = random.sample(resource_provider_traits, 3)
resource_provider_traits_subset_out = traits_to_json(
resource_provider_traits_subset
)
inventory_same_resource_different_traits = chain(
# We are using a provider output with 3 providers, so we need three
# inventory strings - one for each
repeat(inventory_reserved_out, 1), repeat(inventory_list_out, 2)
)
traits_same_resource_different_traits = chain(
# We are using a provider output with 3 providers, so we need three
# trait strings - one for each
repeat(resource_provider_no_traits_out, 1),
repeat(resource_provider_traits_out, 1),
repeat(resource_provider_traits_subset_out, 1)
)
def get_traits_mocked(start=0, stop=None):
def mocked_function(_module, _provider):
all_traits = list(resource_provider_traits)
stop_ = stop if stop else len(all_traits)
return frozenset(all_traits[start:stop_])
return mocked_function
def get_dummy_resources(resource_class="CUSTOM_TEST_RC", amount=3,
traits=[]):
result = [
{
"resource_class": resource_class,
"traits": traits,
"amount": amount
},
]
return result
def get_dummy_module_args(resources,
maximum_retries=10,
delay=10,
venv="/my/dummy/venv"):
result = {
'resources': resources,
'maximum_retries': maximum_retries,
'delay': delay,
'venv': venv
}
return result
def pop_output(a):
next_ = next(a, None)
if next_:
return 0, next_, ""
else:
return 1, "", "Ran out of input"
def noop(*_args, **_kwargs):
pass
def create_run_cmd(providers, inventories, traits):
def dummy_run_command(*args, **_kwargs):
print(args)
if "resource provider list" in args[1]:
return pop_output(providers)
elif "resource provider trait list" in args[1]:
return pop_output(traits)
elif "resource provider inventory list" in args[1]:
return pop_output(inventories)
else:
raise ValueError("{} not expected", args)
return dummy_run_command
class TestTenksWaitForResource(ModuleTestCase):
def setUp(self):
super(TestTenksWaitForResource, self).setUp()
def test_meets_criteria_not_enough(self):
self.assertFalse(meets_criteria(actual=resources_not_enough,
requested=resources_expected))
def test_meets_criteria_one_ok(self):
self.assertFalse(meets_criteria(actual=resources_one_ok,
requested=resources_expected))
def test_meets_criteria_both_ok(self):
self.assertTrue(meets_criteria(actual=resources_both_ok,
requested=resources_expected))
def test_meets_criteria_too_many(self):
self.assertTrue(meets_criteria(actual=resources_too_many,
requested=resources_expected))
def test_meets_criteria_keys_we_dont_care_about(self):
self.assertTrue(meets_criteria(actual=resources_extra_keys,
requested=resources_expected))
def test_meets_criteria_missing_key(self):
self.assertFalse(meets_criteria(actual=resources_key_missing,
requested=resources_expected))
@patch.object(basic.AnsibleModule, 'run_command')
def test_resource_provider_list(self, run_command):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
run_command.return_value = 0, resource_provider_list_out, ''
set_module_args(dummy_module_arguments)
module = wait_for.get_module()
providers = get_providers(module)
calls = [call for call in run_command.call_args_list if
"openstack resource provider list" in call[0][0]]
self.assertGreater(len(calls), 0)
self.assertListEqual(resource_provider_list, providers)
@patch.object(basic.AnsibleModule, 'run_command')
def test_inventory_list(self, run_command):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
run_command.return_value = 0, inventory_list_out, ''
set_module_args(dummy_module_arguments)
module = wait_for.get_module()
expected = get_inventory(module, "provider-uuid")
calls = [call for call in run_command.call_args_list if
"openstack resource provider inventory list" in call[0][0]]
self.assertGreater(len(calls), 0)
self.assertListEqual(expected, inventory_list)
@patch.object(basic.AnsibleModule, 'run_command')
def test_traits(self, run_command):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
run_command.return_value = 0, resource_provider_traits_out, ''
set_module_args(dummy_module_arguments)
module = wait_for.get_module()
expected = get_traits(module, "provider-uuid")
calls = [call for call in run_command.call_args_list if
"openstack resource provider trait list" in call[0][0]]
self.assertGreater(len(calls), 0)
self.assertSetEqual(expected, resource_provider_traits)
def test_venv_path(self):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
module = wait_for.get_module()
openstack_binary_path = get_openstack_binary_path(module)
venv = dummy_module_arguments["venv"]
expected = os.path.join(venv, "bin", "openstack")
self.assertEqual(expected, openstack_binary_path)
def test_venv_path_unset(self):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
args = copy.copy(dummy_module_arguments)
del args["venv"]
set_module_args(args)
module = wait_for.get_module()
openstack_binary_path = get_openstack_binary_path(module)
expected = "openstack"
self.assertEqual(expected, openstack_binary_path)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=cycle(repeat(resource_provider_list_out, 1)),
inventories=cycle(repeat(inventory_reserved_out, 1)),
traits=cycle(repeat(resource_provider_no_traits_out, 3))
))
@patch('time.sleep', noop)
def test_main_failure_exhaust_retries(self):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=dummy_module_arguments["maximum_retries"])
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=cycle(repeat(resource_provider_list_out, 1)),
inventories=cycle(repeat(inventory_list_out, 1)),
traits=cycle(repeat(resource_provider_traits_out, 3))
))
@patch('time.sleep', noop)
@patch.object(wait_for, 'get_traits', get_traits_mocked(0, 2))
def test_main_failure_provider_does_not_provide_all_traits(
self):
expected_traits = list(resource_provider_traits)
resources = get_dummy_resources(traits=expected_traits)
dummy_module_arguments = get_dummy_module_args(
resources=resources,
)
set_module_args(dummy_module_arguments)
expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=dummy_module_arguments["maximum_retries"])
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=cycle(repeat(resource_provider_list_out, 1)),
inventories=cycle(repeat(inventory_list_out, 1)),
traits=cycle(repeat(resource_provider_traits_out, 3))
))
@patch('time.sleep', noop)
def test_main_failure_request_subset_of_traits(
self):
# pick an arbitrary sub range of traits
expected_traits = list(resource_provider_traits)[3:6]
resources = get_dummy_resources(traits=expected_traits)
dummy_module_arguments = get_dummy_module_args(
resources=resources,
)
set_module_args(dummy_module_arguments)
expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=dummy_module_arguments["maximum_retries"])
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=cycle(repeat(resource_provider_list_out, 1)),
inventories=cycle(repeat(inventory_reserved_out, 1)),
traits=cycle(repeat(resource_provider_no_traits_out, 3))
))
@patch('time.sleep', noop)
def test_main_failure_exhaust_retries_traits_not_matched(self):
resources = get_dummy_resources(traits=["WE_NEED_THIS"])
dummy_module_arguments = get_dummy_module_args(
resources=resources
)
set_module_args(dummy_module_arguments)
expected_msg = wait_for._RETRY_LIMIT_FAILURE_TEMPLATE.format(
max_retries=dummy_module_arguments["maximum_retries"])
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 1),
# one per provider
inventories=repeat(inventory_list_out, 3),
traits=repeat(resource_provider_no_traits_out, 3)
))
def test_main_success_one_iteration(self):
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
with self.assertRaises(AnsibleExitJson) as cm:
wait_for.main()
exception = cm.exception
self.assertFalse(exception.args[0]['changed'])
self.assertEqual(1, exception.args[0]["iterations"])
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 1),
inventories=inventory_same_resource_different_traits,
traits=traits_same_resource_different_traits
))
def test_main_success_same_resource_class_different_traits(self):
resource_a = get_dummy_resources(
traits=list(resource_provider_traits_subset),
amount=1
)
resource_b = get_dummy_resources(
traits=list(resource_provider_traits),
amount=1
)
dummy_module_arguments = get_dummy_module_args(
resources=resource_a + resource_b,
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(1, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 1),
# one per provider
inventories=repeat(inventory_list_out, 3),
traits=repeat(resource_provider_no_traits_out, 3)
))
def test_main_success_no_traits(self):
# if the resource provider doesn't have any traits and no traits
# were requested, this should still pass
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(1, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 2),
# one per provider * 2 iterations
inventories=inventory_resource_class_incorrect,
traits=repeat(resource_provider_no_traits_out, 6)))
@patch('time.sleep', noop)
def test_success_first_iteration_wrong_resource_class(self):
# different resource class to the one we are looking for on first
# iteration
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(2, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 2),
inventories=inventory_resource_class_reserved,
# one per provider * 2 iterations
traits=repeat(resource_provider_no_traits_out, 6)))
@patch('time.sleep', noop)
def test_success_resource_class_reserved(self):
# resource class is reserved on the first iteration
dummy_module_arguments = get_dummy_module_args(
resources=dummy_resources
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(2, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
@patch.object(basic.AnsibleModule, 'run_command',
create_run_cmd(
providers=repeat(resource_provider_list_out, 1),
# one per provider
inventories=repeat(inventory_list_out, 3),
traits=repeat(resource_provider_no_traits_out, 3)
))
def test_validation_amount_is_str_repr_of_int(self):
dummy_module_arguments = get_dummy_module_args(
# amount should be an int
resources=get_dummy_resources(amount="3")
)
set_module_args(dummy_module_arguments)
try:
wait_for.main()
except AnsibleExitJson as result:
self.assertFalse(result.args[0]['changed'])
self.assertEqual(1, result.args[0]["iterations"])
else:
self.fail("Should have thrown AnsibleExitJson")
def test_validation_amount_not_int(self):
dummy_module_arguments = get_dummy_module_args(
# amount should be an int
resources=get_dummy_resources(amount="not_a_number")
)
set_module_args(dummy_module_arguments)
expected_msg = "amount, should be type int"
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
def test_validation_amount_missing(self):
resources = get_dummy_resources()
for resource in resources:
del resource["amount"]
dummy_module_arguments = get_dummy_module_args(
resources=resources
)
set_module_args(dummy_module_arguments)
expected_msg = (
"One of your resources does not have the field, amount, set"
)
self.assertRaisesRegex(AnsibleFailJson, expected_msg, wait_for.main)
def test_merge_simple(self):
a = {'a': 1, 'b': 2}
b = {'a': 3, 'c': 5}
expected = {'a': 4, 'b': 2, 'c': 5}
merged = merge(a, b, lambda x, y: x + y)
self.assertDictEqual(expected, merged)

60
tests/utils.py Normal file
View File

@ -0,0 +1,60 @@
import json
import unittest
from ansible.module_utils import basic
from ansible.module_utils._text import to_bytes
# Python 2/3 compatibility.
try:
from unittest.mock import MagicMock, patch
except ImportError:
from mock import MagicMock, patch # noqa
def set_module_args(args):
if '_ansible_remote_tmp' not in args:
args['_ansible_remote_tmp'] = '/tmp'
if '_ansible_keep_remote_files' not in args:
args['_ansible_keep_remote_files'] = False
args = json.dumps({'ANSIBLE_MODULE_ARGS': args})
basic._ANSIBLE_ARGS = to_bytes(args)
class AnsibleExitJson(Exception):
pass
class AnsibleFailJson(Exception):
pass
def exit_json(*args, **kwargs):
if 'changed' not in kwargs:
kwargs['changed'] = False
raise AnsibleExitJson(kwargs)
def fail_json(*args, **kwargs):
kwargs['failed'] = True
raise AnsibleFailJson(kwargs)
class ModuleTestCase(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(ModuleTestCase, self).__init__(*args, **kwargs)
# Python 2 / 3 compatibility. assertRaisesRegexp was renamed to
# assertRaisesRegex in python 3.1.
if hasattr(self, 'assertRaisesRegexp') and \
not hasattr(self, 'assertRaisesRegex'):
self.assertRaisesRegex = self.assertRaisesRegexp
def setUp(self):
self.mock_module = patch.multiple(basic.AnsibleModule,
exit_json=exit_json,
fail_json=fail_json)
self.mock_module.start()
set_module_args({})
self.addCleanup(self.mock_module.stop)

2
tox.ini
View File

@ -49,7 +49,7 @@ commands = {posargs}
basepython = python3
setenv =
VIRTUAL_ENV={envdir}
PYTHON=coverage run --source tenks,ansible/action_plugins --parallel-mode
PYTHON=coverage run --source tenks,ansible --parallel-mode
commands =
coverage erase
stestr run {posargs}