openstack/kayobe
Code Issues Proposed changes

Add unit tests and style checks for Kayobe python module

Browse Source 
Unit tests can be run via:

tox -e py27

Style checks can be run via:

tox -e pep8
This commit is contained in:
Mark Goddard 2017-03-01 13:32:34 +00:00
parent caf9b52ac7
commit baf17c8cff
13 changed files with 475 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.gitignore vendored
View File

@ -23,3 +23,12 @@ ansible/roles/yatesr.timezone/
# Virtualenv
ansible/kolla-venv
# Tox
.tox/
# Python build artifacts
kayobe
# Python build artifacts
kayobe.egg-info

42
kayobe/ansible.py
View File

@ -16,19 +16,6 @@ CONFIG_PATH_ENV = "KAYOBE_CONFIG_PATH"
LOG = logging.getLogger(__name__)
def galaxy_install(role_file, roles_path):
"""Install Ansible roles via Ansible Galaxy."""
cmd = ["ansible-galaxy", "install"]
cmd += ["--roles-path", roles_path]
cmd += ["--role-file", role_file]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as e:
LOG.error("Failed to install Ansible roles from %s via Ansible "
"Galaxy: returncode %d", role_file, e.returncode)
sys.exit(e.returncode)
def add_args(parser):
"""Add arguments required for running Ansible playbooks to a parser."""
default_config_path = os.getenv(CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH)
@ -53,9 +40,9 @@ def add_args(parser):
parser.add_argument("-l", "--limit", metavar="SUBSET",
help="further limit selected hosts to an additional "
"pattern")
parser.add_argument("-t", "--tags", metavar="TAGS", action="append",
parser.add_argument("-t", "--tags", metavar="TAGS",
help="only run plays and tasks tagged with these "
"values")
"values")
def _get_inventory_path(parsed_args):
@ -75,7 +62,7 @@ def _validate_args(parsed_args, playbooks):
sys.exit(1)
inventory = _get_inventory_path(parsed_args)
result = utils.is_readable_file(inventory)
result = utils.is_readable_dir(inventory)
if not result["result"]:
LOG.error("Kayobe inventory %s is invalid: %s",
inventory, result["message"])
@ -89,18 +76,27 @@ def _validate_args(parsed_args, playbooks):
sys.exit(1)
def _get_vars_files(config_path):
"""Return a list of Kayobe Ansible configuration variable files."""
vars_files = []
for vars_file in os.listdir(config_path):
abs_path = os.path.join(config_path, vars_file)
if utils.is_readable_file(abs_path):
root, ext = os.path.splitext(vars_file)
if ext in (".yml", ".yaml", ".json"):
vars_files.append(abs_path)
return vars_files
def build_args(parsed_args, playbooks,
extra_vars=None, limit=None, tags=None):
"""Build arguments required for running Ansible playbooks."""
cmd = ["ansible-playbook"]
inventory = _get_inventory_path(parsed_args)
cmd += ["--inventory", inventory]
for vars_file in os.listdir(parsed_args.config_path):
abs_path = os.path.join(parsed_args.config_path, vars_file)
if os.path.isfile(abs_path):
root, ext = os.path.splitext(vars_file)
if ext in (".yml", ".yaml", ".json"):
cmd += ["-e", "@%s" % abs_path]
vars_files = _get_vars_files(parsed_args.config_path)
for vars_file in vars_files:
cmd += ["-e", "@%s" % vars_file]
if parsed_args.extra_vars:
for extra_var in parsed_args.extra_vars:
cmd += ["-e", extra_var]
@ -141,7 +137,7 @@ def run_playbook(parsed_args, playbook, *args, **kwargs):
def config_dump(parsed_args, host=None, hosts=None, var_name=None,
facts=False, extra_vars=None):
facts=None, extra_vars=None):
dump_dir = tempfile.mkdtemp()
try:
if not extra_vars:

2
kayobe/cli/commands.py
View File

@ -47,7 +47,7 @@ class ControlHostBootstrap(KayobeAnsibleMixin, Command):
self.app.LOG.error("%s is not currently supported", linux_distname)
sys.exit(1)
utils.yum_install(["ansible"])
ansible.galaxy_install("ansible/requirements.yml", "ansible/roles")
utils.galaxy_install("ansible/requirements.yml", "ansible/roles")
playbooks = ["ansible/%s.yml" % playbook for playbook in
"bootstrap", "kolla"]
ansible.run_playbooks(parsed_args, playbooks)

15
kayobe/kolla_ansible.py
View File

@ -26,16 +26,16 @@ def add_args(parser):
help="path to Kolla configuration. "
"(default=$%s or %s)" %
(CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH))
parser.add_argument("--kolla-extra-vars", metavar="EXTRA_VARS",
parser.add_argument("-ke", "--kolla-extra-vars", metavar="EXTRA_VARS",
action="append",
help="set additional variables as key=value or "
"YAML/JSON for Kolla Ansible")
parser.add_argument("--kolla-inventory", metavar="INVENTORY",
parser.add_argument("-ki", "--kolla-inventory", metavar="INVENTORY",
help="specify inventory host path "
"(default=$%s/inventory or %s/inventory) or "
"comma-separated host list for Kolla Ansible" %
(CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH))
parser.add_argument("--kolla-tags", metavar="TAGS", action="append",
parser.add_argument("-kt", "--kolla-tags", metavar="TAGS",
help="only run plays and tasks tagged with these "
"values in Kolla Ansible")
parser.add_argument("--kolla-venv", metavar="VENV", default=default_venv,
@ -79,12 +79,13 @@ def build_args(parsed_args, command, inventory_filename, extra_vars=None,
"""Build arguments required for running Kolla Ansible."""
venv_activate = os.path.join(parsed_args.kolla_venv, "bin", "activate")
cmd = ["source", venv_activate, "&&"]
cmd = ["kolla-ansible", command]
cmd += ["kolla-ansible", command]
inventory = _get_inventory_path(parsed_args, inventory_filename)
cmd += ["--inventory", inventory]
cmd += ["--configdir", parsed_args.kolla_config_path]
cmd += ["--passwords",
os.path.join(parsed_args.kolla_config_path, "passwords.yml")]
if parsed_args.kolla_config_path != DEFAULT_CONFIG_PATH:
cmd += ["--configdir", parsed_args.kolla_config_path]
cmd += ["--passwords",
os.path.join(parsed_args.kolla_config_path, "passwords.yml")]
if parsed_args.kolla_extra_vars:
for extra_var in parsed_args.kolla_extra_vars:
cmd += ["-e", extra_var]

0
kayobe/tests/__init__.py Normal file
View File

0
kayobe/tests/unit/__init__.py Normal file
View File

188
kayobe/tests/unit/test_ansible.py Normal file
View File

@ -0,0 +1,188 @@
import argparse
import os
import shutil
import subprocess
import tempfile
import unittest
import mock
from kayobe import ansible
from kayobe import utils
class TestCase(unittest.TestCase):
@mock.patch.object(utils, "run_command")
@mock.patch.object(ansible, "_get_vars_files")
@mock.patch.object(ansible, "_validate_args")
def test_run_playbooks(self, mock_validate, mock_vars, mock_run):
mock_vars.return_value = ["/etc/kayobe/vars-file1.yml",
"/etc/kayobe/vars-file2.yaml"]
parser = argparse.ArgumentParser()
ansible.add_args(parser)
parsed_args = parser.parse_args([])
ansible.run_playbooks(parsed_args, ["playbook1.yml", "playbook2.yml"])
expected_cmd = [
"ansible-playbook",
"--inventory", "/etc/kayobe/inventory",
"-e", "@/etc/kayobe/vars-file1.yml",
"-e", "@/etc/kayobe/vars-file2.yaml",
"playbook1.yml",
"playbook2.yml",
]
mock_run.assert_called_once_with(expected_cmd, quiet=False)
mock_vars.assert_called_once_with("/etc/kayobe")
@mock.patch.object(utils, "run_command")
@mock.patch.object(ansible, "_get_vars_files")
@mock.patch.object(ansible, "_validate_args")
def test_run_playbooks_all_the_args(self, mock_validate, mock_vars,
mock_run):
mock_vars.return_value = ["/path/to/config/vars-file1.yml",
"/path/to/config/vars-file2.yaml"]
parser = argparse.ArgumentParser()
ansible.add_args(parser)
args = [
"-b",
"-C",
"--config-path", "/path/to/config",
"-e", "ev_name1=ev_value1",
"-i", "/path/to/inventory",
"-l", "group1:host",
"-t", "tag1,tag2",
]
parsed_args = parser.parse_args(args)
ansible.run_playbooks(parsed_args, ["playbook1.yml", "playbook2.yml"])
expected_cmd = [
"ansible-playbook",
"--inventory", "/path/to/inventory",
"-e", "@/path/to/config/vars-file1.yml",
"-e", "@/path/to/config/vars-file2.yaml",
"-e", "ev_name1=ev_value1",
"--become",
"--check",
"--limit", "group1:host",
"--tags", "tag1,tag2",
"playbook1.yml",
"playbook2.yml",
]
mock_run.assert_called_once_with(expected_cmd, quiet=False)
mock_vars.assert_called_once_with("/path/to/config")
@mock.patch.object(utils, "run_command")
@mock.patch.object(ansible, "_get_vars_files")
@mock.patch.object(ansible, "_validate_args")
def test_run_playbooks_all_the_long_args(self, mock_validate, mock_vars,
mock_run):
mock_vars.return_value = ["/path/to/config/vars-file1.yml",
"/path/to/config/vars-file2.yaml"]
parser = argparse.ArgumentParser()
ansible.add_args(parser)
args = [
"--become",
"--check",
"--config-path", "/path/to/config",
"--extra-vars", "ev_name1=ev_value1",
"--inventory", "/path/to/inventory",
"--limit", "group1:host1",
"--tags", "tag1,tag2",
]
parsed_args = parser.parse_args(args)
ansible.run_playbooks(parsed_args, ["playbook1.yml", "playbook2.yml"])
expected_cmd = [
"ansible-playbook",
"--inventory", "/path/to/inventory",
"-e", "@/path/to/config/vars-file1.yml",
"-e", "@/path/to/config/vars-file2.yaml",
"-e", "ev_name1=ev_value1",
"--become",
"--check",
"--limit", "group1:host1",
"--tags", "tag1,tag2",
"playbook1.yml",
"playbook2.yml",
]
mock_run.assert_called_once_with(expected_cmd, quiet=False)
mock_vars.assert_called_once_with("/path/to/config")
@mock.patch.object(utils, "run_command")
@mock.patch.object(ansible, "_get_vars_files")
@mock.patch.object(ansible, "_validate_args")
def test_run_playbooks_func_args(self, mock_validate, mock_vars, mock_run):
mock_vars.return_value = ["/etc/kayobe/vars-file1.yml",
"/etc/kayobe/vars-file2.yaml"]
parser = argparse.ArgumentParser()
ansible.add_args(parser)
args = [
"--extra-vars", "ev_name1=ev_value1",
"--limit", "group1:host1",
"--tags", "tag1,tag2",
]
parsed_args = parser.parse_args(args)
kwargs = {
"extra_vars": {"ev_name2": "ev_value2"},
"limit": "group2:host2",
"tags": "tag3,tag4",
}
ansible.run_playbooks(parsed_args, ["playbook1.yml", "playbook2.yml"],
**kwargs)
expected_cmd = [
"ansible-playbook",
"--inventory", "/etc/kayobe/inventory",
"-e", "@/etc/kayobe/vars-file1.yml",
"-e", "@/etc/kayobe/vars-file2.yaml",
"-e", "ev_name1=ev_value1",
"-e", "ev_name2=ev_value2",
"--limit", "group1:host1&group2:host2",
"--tags", "tag1,tag2,tag3,tag4",
"playbook1.yml",
"playbook2.yml",
]
mock_run.assert_called_once_with(expected_cmd, quiet=False)
mock_vars.assert_called_once_with("/etc/kayobe")
@mock.patch.object(utils, "run_command")
@mock.patch.object(ansible, "_get_vars_files")
@mock.patch.object(ansible, "_validate_args")
def test_run_playbooks_failure(self, mock_validate, mock_vars, mock_run):
parser = argparse.ArgumentParser()
ansible.add_args(parser)
parsed_args = parser.parse_args([])
mock_run.side_effect = subprocess.CalledProcessError(1, "dummy")
self.assertRaises(SystemExit,
ansible.run_playbooks, parsed_args, "command")
@mock.patch.object(shutil, 'rmtree')
@mock.patch.object(utils, 'read_yaml_file')
@mock.patch.object(os, 'listdir')
@mock.patch.object(ansible, 'run_playbook')
@mock.patch.object(tempfile, 'mkdtemp')
def test_config_dump(self, mock_mkdtemp, mock_run, mock_listdir, mock_read,
mock_rmtree):
parser = argparse.ArgumentParser()
parsed_args = parser.parse_args([])
dump_dir = mock_mkdtemp.return_value
mock_listdir.return_value = ["host1.yml", "host2.yml"]
mock_read.side_effect = [
{"var1": "value1"},
{"var2": "value2"}
]
result = ansible.config_dump(parsed_args)
expected_result = {
"host1": {"var1": "value1"},
"host2": {"var2": "value2"},
}
self.assertEqual(result, expected_result)
mock_run.assert_called_once_with(parsed_args,
"ansible/dump-config.yml",
extra_vars={
"dump_path": dump_dir,
},
quiet=True)
mock_rmtree.assert_called_once_with(dump_dir)
mock_listdir.assert_called_once_with(dump_dir)
mock_read.assert_has_calls([
mock.call(os.path.join(dump_dir, "host1.yml")),
mock.call(os.path.join(dump_dir, "host2.yml")),
])

113
kayobe/tests/unit/test_kolla_ansible.py Normal file
View File

@ -0,0 +1,113 @@
import argparse
import subprocess
import unittest
import mock
from kayobe import kolla_ansible
from kayobe import utils
class TestCase(unittest.TestCase):
@mock.patch.object(utils, "run_command")
@mock.patch.object(kolla_ansible, "_validate_args")
def test_run(self, mock_validate, mock_run):
parser = argparse.ArgumentParser()
kolla_ansible.add_args(parser)
parsed_args = parser.parse_args([])
kolla_ansible.run(parsed_args, "command", "overcloud")
expected_cmd = [
"source", "ansible/kolla-venv/bin/activate", "&&",
"kolla-ansible", "command",
"--inventory", "/etc/kolla/inventory/overcloud",
]
expected_cmd = " ".join(expected_cmd)
mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False)
@mock.patch.object(utils, "run_command")
@mock.patch.object(kolla_ansible, "_validate_args")
def test_run_all_the_args(self, mock_validate, mock_run):
parser = argparse.ArgumentParser()
kolla_ansible.add_args(parser)
args = [
"--kolla-config-path", "/path/to/config",
"-ke", "ev_name1=ev_value1",
"-ki", "/path/to/inventory",
"-kt", "tag1,tag2",
]
parsed_args = parser.parse_args(args)
kolla_ansible.run(parsed_args, "command", "overcloud")
expected_cmd = [
"source", "ansible/kolla-venv/bin/activate", "&&",
"kolla-ansible", "command",
"--inventory", "/path/to/inventory",
"--configdir", "/path/to/config",
"--passwords", "/path/to/config/passwords.yml",
"-e", "ev_name1=ev_value1",
"--tags", "tag1,tag2",
]
expected_cmd = " ".join(expected_cmd)
mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False)
@mock.patch.object(utils, "run_command")
@mock.patch.object(kolla_ansible, "_validate_args")
def test_run_all_the_long_args(self, mock_validate, mock_run):
parser = argparse.ArgumentParser()
kolla_ansible.add_args(parser)
args = [
"--kolla-config-path", "/path/to/config",
"--kolla-extra-vars", "ev_name1=ev_value1",
"--kolla-inventory", "/path/to/inventory",
"--kolla-tags", "tag1,tag2",
]
parsed_args = parser.parse_args(args)
kolla_ansible.run(parsed_args, "command", "overcloud")
expected_cmd = [
"source", "ansible/kolla-venv/bin/activate", "&&",
"kolla-ansible", "command",
"--inventory", "/path/to/inventory",
"--configdir", "/path/to/config",
"--passwords", "/path/to/config/passwords.yml",
"-e", "ev_name1=ev_value1",
"--tags", "tag1,tag2",
]
expected_cmd = " ".join(expected_cmd)
mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False)
@mock.patch.object(utils, "run_command")
@mock.patch.object(kolla_ansible, "_validate_args")
def test_run_func_args(self, mock_validate, mock_run):
parser = argparse.ArgumentParser()
kolla_ansible.add_args(parser)
args = [
"--kolla-extra-vars", "ev_name1=ev_value1",
"--kolla-tags", "tag1,tag2",
]
parsed_args = parser.parse_args(args)
kwargs = {
"extra_vars": {"ev_name2": "ev_value2"},
"tags": "tag3,tag4",
}
kolla_ansible.run(parsed_args, "command", "overcloud", **kwargs)
expected_cmd = [
"source", "ansible/kolla-venv/bin/activate", "&&",
"kolla-ansible", "command",
"--inventory", "/etc/kolla/inventory/overcloud",
"-e", "ev_name1=ev_value1",
"-e", "ev_name2=ev_value2",
"--tags", "tag1,tag2,tag3,tag4",
]
expected_cmd = " ".join(expected_cmd)
mock_run.assert_called_once_with(expected_cmd, shell=True, quiet=False)
@mock.patch.object(utils, "run_command")
@mock.patch.object(kolla_ansible, "_validate_args")
def test_run_failure(self, mock_validate, mock_run):
parser = argparse.ArgumentParser()
kolla_ansible.add_args(parser)
parsed_args = parser.parse_args([])
mock_run.side_effect = subprocess.CalledProcessError(1, "dummy")
self.assertRaises(SystemExit,
kolla_ansible.run, parsed_args, "command",
"overcloud")

73
kayobe/tests/unit/test_utils.py Normal file
View File

@ -0,0 +1,73 @@
import subprocess
import unittest
import mock
from kayobe import utils
class TestCase(unittest.TestCase):
@mock.patch.object(utils, "run_command")
def test_yum_install(self, mock_run):
utils.yum_install(["package1", "package2"])
mock_run.assert_called_once_with(["sudo", "yum", "-y", "install",
"package1", "package2"])
@mock.patch.object(utils, "run_command")
def test_yum_install_failure(self, mock_run):
mock_run.side_effect = subprocess.CalledProcessError(1, "command")
self.assertRaises(SystemExit,
utils.yum_install, ["package1", "package2"])
@mock.patch.object(utils, "run_command")
def test_galaxy_install(self, mock_run):
utils.galaxy_install("/path/to/role/file", "/path/to/roles")
mock_run.assert_called_once_with(["ansible-galaxy", "install",
"--roles-path", "/path/to/roles",
"--role-file", "/path/to/role/file"])
@mock.patch.object(utils, "run_command")
def test_galaxy_install_failure(self, mock_run):
mock_run.side_effect = subprocess.CalledProcessError(1, "command")
self.assertRaises(SystemExit,
utils.galaxy_install, "/path/to/role/file",
"/path/to/roles")
@mock.patch.object(utils, "read_file")
def test_read_yaml_file(self, mock_read):
mock_read.return_value = """---
key1: value1
key2: value2
"""
result = utils.read_yaml_file("/path/to/file")
self.assertEqual(result, {"key1": "value1", "key2": "value2"})
mock_read.assert_called_once_with("/path/to/file")
@mock.patch.object(utils, "read_file")
def test_read_yaml_file_open_failure(self, mock_read):
mock_read.side_effect = IOError
self.assertRaises(SystemExit, utils.read_yaml_file, "/path/to/file")
@mock.patch.object(utils, "read_file")
def test_read_yaml_file_not_yaml(self, mock_read):
mock_read.return_value = "[1{!"
self.assertRaises(SystemExit, utils.read_yaml_file, "/path/to/file")
@mock.patch.object(subprocess, "check_call")
def test_run_command(self, mock_call):
utils.run_command(["command", "to", "run"])
mock_call.assert_called_once_with(["command", "to", "run"])
@mock.patch.object(subprocess, "check_call")
def test_run_command_quiet(self, mock_call):
utils.run_command(["command", "to", "run"], quiet=True)
mock_call.assert_called_once_with(["command", "to", "run"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
@mock.patch.object(subprocess, "check_call")
def test_run_command_failure(self, mock_call):
mock_call.side_effect = subprocess.CalledProcessError(1, "command")
self.assertRaises(subprocess.CalledProcessError, utils.run_command,
["command", "to", "run"])

27
kayobe/utils.py
View File

@ -2,6 +2,7 @@ import logging
import os
import subprocess
import sys
import yaml
@ -13,25 +14,43 @@ def yum_install(packages):
cmd = ["sudo", "yum", "-y", "install"]
cmd += packages
try:
subprocess.check_call(cmd)
run_command(cmd)
except subprocess.CalledProcessError as e:
print ("Failed to install packages %s via Yum: returncode %d" %
(", ".join(packages), e.returncode))
sys.exit(e.returncode)
def galaxy_install(role_file, roles_path):
"""Install Ansible roles via Ansible Galaxy."""
cmd = ["ansible-galaxy", "install"]
cmd += ["--roles-path", roles_path]
cmd += ["--role-file", role_file]
try:
run_command(cmd)
except subprocess.CalledProcessError as e:
LOG.error("Failed to install Ansible roles from %s via Ansible "
"Galaxy: returncode %d", role_file, e.returncode)
sys.exit(e.returncode)
def read_file(path, mode="r"):
"""Read the content of a file."""
with open(path, mode) as f:
return f.read()
def read_yaml_file(path):
"""Read and decode a YAML file."""
try:
with open(path, "r") as f:
content = f.read()
content = read_file(path)
except IOError as e:
print ("Failed to open config dump file %s: %s" %
(path, repr(e)))
sys.exit(1)
try:
return yaml.load(content)
except ValueError as e:
except yaml.YAMLError as e:
print ("Failed to decode config dump YAML file %s: %s" %
(path, repr(e)))
sys.exit(1)

1
requirements.txt
View File

@ -1 +1,2 @@
cliff
PyYAML

5
test-requirements.txt Normal file
View File

@ -0,0 +1,5 @@
hacking
coverage
flake8-import-order
mock
unittest2

35
tox.ini Normal file
View File

@ -0,0 +1,35 @@
[tox]
minversion = 1.8
skipsdist = True
envlist = py27,pep8
[testenv]
usedevelop = True
install_command = pip install -U {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
PYTHONDONTWRITEBYTECODE = 1
LANGUAGE=en_US
LC_ALL=en_US.UTF-8
PYTHONWARNINGS=default::DeprecationWarning
TESTS_DIR=./kayobe/tests/unit/
deps = -r{toxinidir}/test-requirements.txt
commands = unit2 discover {posargs}
passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
[testenv:pep8]
whitelist_externals = bash
commands =
flake8 {posargs}
[testenv:venv]
setenv = PYTHONHASHSEED=0
commands = {posargs}
[flake8]
exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build
import-order-style = pep8
max-complexity=17
# [H106] Don’t put vim configuration in source files.
# [H203] Use assertIs(Not)None to check for None.
# [H904] Delay string interpolations at logging calls.
enable-extensions=H106,H203,H904