Add kayobe python module with CLI

The CLI replaces all existing shell scripts except for
configure-kayobe.sh. Other shell scripts are now removed.

bootstrap.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-set -e
-
-function run_playbook {
-    KAYOBE_CONFIG_PATH=${KAYOBE_CONFIG_PATH:-/etc/kayobe}
-    # Ansible fails silently if the inventory does not exist.
-    test -e ${KAYOBE_CONFIG_PATH}/inventory
-    ansible-playbook \
-        -i ${KAYOBE_CONFIG_PATH}/inventory \
-        -e @${KAYOBE_CONFIG_PATH}/globals.yml \
-        -e @${KAYOBE_CONFIG_PATH}/dns.yml \
-        -e @${KAYOBE_CONFIG_PATH}/kolla.yml \
-        -e @${KAYOBE_CONFIG_PATH}/networks.yml \
-        -e @${KAYOBE_CONFIG_PATH}/network-allocation.yml \
-        -e @${KAYOBE_CONFIG_PATH}/ntp.yml \
-        -e @${KAYOBE_CONFIG_PATH}/swift.yml \
-        $@
-}
-
-function install_ansible {
-    if [[ -f /etc/centos-release ]]; then
-        sudo yum -y install epel-release
-    elif [[ -f /etc/redhat-release ]]; then
-        sudo subscription-manager repos --enable=qci-1.0-for-rhel-7-rpms
-        if ! yum info epel-release >/dev/null 2>&1 ; then
-            sudo yum -y install \
-                https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-        fi
-    fi
-    sudo yum -y install ansible
-}
-
-function install_ansible_roles {
-    ansible-galaxy install \
-        --roles-path ansible/roles \
-        --role-file ansible/requirements.yml
-}
-
-function bootstrap {
-    run_playbook ansible/bootstrap.yml
-}
-
-function install_kolla {
-    run_playbook ansible/kolla.yml
-}
-
-function main {
-    install_ansible
-    install_ansible_roles
-    bootstrap
-    install_kolla
-}
-
-main $*

deploy-overcloud.sh

@@ -1,71 +0,0 @@
-#!/bin/bash
-
-set -e
-
-function run_playbook {
-    KAYOBE_CONFIG_PATH=${KAYOBE_CONFIG_PATH:-/etc/kayobe}
-    # Ansible fails silently if the inventory does not exist.
-    test -e ${KAYOBE_CONFIG_PATH}/inventory
-    ansible-playbook \
-        -i ${KAYOBE_CONFIG_PATH}/inventory \
-        -e @${KAYOBE_CONFIG_PATH}/controllers.yml \
-        -e @${KAYOBE_CONFIG_PATH}/dns.yml \
-        -e @${KAYOBE_CONFIG_PATH}/globals.yml \
-        -e @${KAYOBE_CONFIG_PATH}/kolla.yml \
-        -e @${KAYOBE_CONFIG_PATH}/networks.yml \
-        -e @${KAYOBE_CONFIG_PATH}/network-allocation.yml \
-        -e @${KAYOBE_CONFIG_PATH}/ntp.yml \
-        -e @${KAYOBE_CONFIG_PATH}/ssh.yml \
-        -e @${KAYOBE_CONFIG_PATH}/swift.yml \
-        $@
-}
-
-function run_kolla_ansible {
-    export KOLLA_CONFIG_PATH=${KOLLA_CONFIG_PATH:-/etc/kolla}
-    # Ansible fails silently if the inventory does not exist.
-    test -e ${KOLLA_CONFIG_PATH}/inventory/overcloud
-    KOLLA_VENV=$(pwd)/ansible/kolla-venv
-    source ${KOLLA_VENV}/bin/activate
-    kolla-ansible \
-        --configdir ${KOLLA_CONFIG_PATH} \
-        --passwords ${KOLLA_CONFIG_PATH}/passwords.yml \
-        -i ${KOLLA_CONFIG_PATH}/inventory/overcloud \
-        $@
-    deactivate
-}
-
-function configure_os {
-    ansible_user=$(./kayobe-config-dump -e dump_hosts=controllers[0] -e dump_var_name=kayobe_ansible_user | head -n -1)
-    run_playbook ansible/ip-allocation.yml -l controllers
-    run_playbook ansible/ssh-known-host.yml -l controllers
-    run_playbook ansible/kayobe-ansible-user.yml -l controllers
-    run_playbook ansible/disable-selinux.yml -l controllers
-    run_playbook ansible/network.yml -l controllers
-    run_playbook ansible/ntp.yml -l controllers
-    run_kolla_ansible bootstrap-servers -e ansible_user=${ansible_user}
-    run_playbook ansible/kolla-host.yml -l controllers
-    run_playbook ansible/docker.yml -l controllers
-}
-
-function deploy_services {
-    run_playbook ansible/kolla-openstack.yml
-    run_playbook ansible/swift-setup.yml
-    run_kolla_ansible pull
-    run_kolla_ansible prechecks
-    run_kolla_ansible deploy
-    run_kolla_ansible post-deploy -e node_config_directory=${KOLLA_CONFIG_PATH}
-}
-
-function deploy_overcloud {
-    configure_os
-    deploy_services
-}
-
-###########################################################
-# Main
-
-function main {
-    deploy_overcloud
-}
-
-deploy_overcloud

deploy-seed.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-
-set -e
-
-function run_playbook {
-    KAYOBE_CONFIG_PATH=${KAYOBE_CONFIG_PATH:-/etc/kayobe}
-    # Ansible fails silently if the inventory does not exist.
-    test -e ${KAYOBE_CONFIG_PATH}/inventory
-    ansible-playbook \
-        -i ${KAYOBE_CONFIG_PATH}/inventory \
-        -e @${KAYOBE_CONFIG_PATH}/bifrost.yml \
-        -e @${KAYOBE_CONFIG_PATH}/dns.yml \
-        -e @${KAYOBE_CONFIG_PATH}/globals.yml \
-        -e @${KAYOBE_CONFIG_PATH}/kolla.yml \
-        -e @${KAYOBE_CONFIG_PATH}/networks.yml \
-        -e @${KAYOBE_CONFIG_PATH}/network-allocation.yml \
-        -e @${KAYOBE_CONFIG_PATH}/ntp.yml \
-        -e @${KAYOBE_CONFIG_PATH}/seed-vm.yml \
-        -e @${KAYOBE_CONFIG_PATH}/ssh.yml \
-        -e @${KAYOBE_CONFIG_PATH}/swift.yml \
-        $@
-}
-
-function run_kolla_ansible {
-    export KOLLA_CONFIG_PATH=${KOLLA_CONFIG_PATH:-/etc/kolla}
-    # Ansible fails silently if the inventory does not exist.
-    test -e ${KOLLA_CONFIG_PATH}/inventory/seed
-    KOLLA_VENV=$(pwd)/ansible/kolla-venv
-    source ${KOLLA_VENV}/bin/activate
-    kolla-ansible \
-        --configdir ${KOLLA_CONFIG_PATH} \
-        --passwords ${KOLLA_CONFIG_PATH}/passwords.yml \
-        -i ${KOLLA_CONFIG_PATH}/inventory/seed \
-        $@
-    deactivate
-}
-
-function configure_os {
-    ansible_user=$(./kayobe-config-dump -e dump_hosts=seed -e dump_var_name=kayobe_ansible_user | head -n -1)
-    run_playbook ansible/ip-allocation.yml -l seed
-    run_playbook ansible/ssh-known-host.yml -l seed
-    run_playbook ansible/kayobe-ansible-user.yml -l seed
-    run_playbook ansible/disable-selinux.yml -l seed
-    run_playbook ansible/network.yml -l seed
-    run_playbook ansible/ntp.yml -l seed
-    run_kolla_ansible bootstrap-servers -e ansible_user=${ansible_user}
-    run_playbook ansible/kolla-host.yml -l seed
-    run_playbook ansible/docker.yml -l seed
-}
-
-function deploy_bifrost {
-    # Use a pre-built bifrost image in the stackhpc repository.
-    # The image was built via kolla-build -t source bifrost-deploy.
-    run_playbook ansible/kolla-bifrost.yml
-    run_kolla_ansible deploy-bifrost \
-      -e kolla_install_type=source \
-      -e docker_namespace=stackhpc
-}
-
-function deploy_seed_node {
-    configure_os
-    deploy_bifrost
-}
-
-###########################################################
-# Main
-
-function main {
-    deploy_seed_node
-}
-
-main $*

kayobe-config-dump

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-dump_dir=$(mktemp -d)
-
-# Execute the dump-config.yml playbook.
-./kayobe-playbook \
-    ansible/dump-config.yml \
-    -e dump_path=${dump_dir} \
-    $@ \
-    > /dev/null
-
-result=$?
-
-if [[ $result -eq 0 ]]; then
-    cat ${dump_dir}/*.yml
-fi
-rm -rf ${dump_dir}
-exit $result

kayobe-playbook

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-KAYOBE_CONFIG_PATH=${KAYOBE_CONFIG_PATH:-/etc/kayobe}
-
-# Ansible fails silently if the inventory does not exist.
-test -e ${KAYOBE_CONFIG_PATH}/inventory
-
-# Execute a Kayobe playbook.
-exec ansible-playbook \
-  -i ${KAYOBE_CONFIG_PATH}/inventory \
-  -e @${KAYOBE_CONFIG_PATH}/bifrost.yml \
-  -e @${KAYOBE_CONFIG_PATH}/controllers.yml \
-  -e @${KAYOBE_CONFIG_PATH}/dns.yml \
-  -e @${KAYOBE_CONFIG_PATH}/globals.yml \
-  -e @${KAYOBE_CONFIG_PATH}/kolla.yml \
-  -e @${KAYOBE_CONFIG_PATH}/networks.yml \
-  -e @${KAYOBE_CONFIG_PATH}/network-allocation.yml \
-  -e @${KAYOBE_CONFIG_PATH}/ntp.yml \
-  -e @${KAYOBE_CONFIG_PATH}/seed-vm.yml \
-  -e @${KAYOBE_CONFIG_PATH}/ssh.yml \
-  -e @${KAYOBE_CONFIG_PATH}/swift.yml \
-  $@

kayobe/ansible.py

@@ -0,0 +1,173 @@
+import logging
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import tempfile
+
+from kayobe import utils
+
+
+DEFAULT_CONFIG_PATH = "/etc/kayobe"
+
+CONFIG_PATH_ENV = "KAYOBE_CONFIG_PATH"
+
+LOG = logging.getLogger(__name__)
+
+
+def galaxy_install(role_file, roles_path):
+    """Install Ansible roles via Ansible Galaxy."""
+    cmd = ["ansible-galaxy", "install"]
+    cmd += ["--roles-path", roles_path]
+    cmd += ["--role-file", role_file]
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        LOG.error("Failed to install Ansible roles from %s via Ansible "
+                  "Galaxy: returncode %d", role_file, e.returncode)
+        sys.exit(e.returncode)
+
+
+def add_args(parser):
+    """Add arguments required for running Ansible playbooks to a parser."""
+    default_config_path = os.getenv(CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH)
+    parser.add_argument("-b", "--become", action="store_true",
+                        help="run operations with become (nopasswd implied)")
+    parser.add_argument("-C", "--check", action="store_true",
+                        help="don't make any changes; instead, try to predict "
+                             "some of the changes that may occur")
+    parser.add_argument("--config-path", default=default_config_path,
+                        help="path to Kayobe configuration. "
+                             "(default=$%s or %s)" %
+                             (CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH))
+    parser.add_argument("-e", "--extra-vars", metavar="EXTRA_VARS",
+                        action="append",
+                        help="set additional variables as key=value or "
+                             "YAML/JSON")
+    parser.add_argument("-i", "--inventory", metavar="INVENTORY",
+                        help="specify inventory host path "
+                             "(default=$%s/inventory or %s/inventory) or "
+                             "comma-separated host list" %
+                             (CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH))
+    parser.add_argument("-l", "--limit", metavar="SUBSET",
+                        help="further limit selected hosts to an additional "
+                             "pattern")
+    parser.add_argument("-t", "--tags", metavar="TAGS", action="append",
+                        help="only run plays and tasks tagged with these "
+                        "values")
+
+
+def _get_inventory_path(parsed_args):
+    """Return the path to the Kayobe inventory."""
+    if parsed_args.inventory:
+        return parsed_args.inventory
+    else:
+        return os.path.join(parsed_args.config_path, "inventory")
+
+
+def _validate_args(parsed_args, playbooks):
+    """Validate Kayobe Ansible arguments."""
+    result = utils.is_readable_dir(parsed_args.config_path)
+    if not result["result"]:
+        LOG.error("Kayobe configuration path %s is invalid: %s",
+                  parsed_args.config_path, result["message"])
+        sys.exit(1)
+
+    inventory = _get_inventory_path(parsed_args)
+    result = utils.is_readable_file(inventory)
+    if not result["result"]:
+        LOG.error("Kayobe inventory %s is invalid: %s",
+                  inventory, result["message"])
+        sys.exit(1)
+
+    for playbook in playbooks:
+        result = utils.is_readable_file(playbook)
+        if not result["result"]:
+            LOG.error("Kayobe playbook %s is invalid: %s",
+                      playbook, result["message"])
+            sys.exit(1)
+
+
+def build_args(parsed_args, playbooks,
+               extra_vars=None, limit=None, tags=None):
+    """Build arguments required for running Ansible playbooks."""
+    cmd = ["ansible-playbook"]
+    inventory = _get_inventory_path(parsed_args)
+    cmd += ["--inventory", inventory]
+    for vars_file in os.listdir(parsed_args.config_path):
+        abs_path = os.path.join(parsed_args.config_path, vars_file)
+        if os.path.isfile(abs_path):
+            root, ext = os.path.splitext(vars_file)
+            if ext in (".yml", ".yaml", ".json"):
+                cmd += ["-e", "@%s" % abs_path]
+    if parsed_args.extra_vars:
+        for extra_var in parsed_args.extra_vars:
+            cmd += ["-e", extra_var]
+    if extra_vars:
+        for extra_var_name, extra_var_value in extra_vars.items():
+            cmd += ["-e", "%s=%s" % (extra_var_name, extra_var_value)]
+    if parsed_args.become:
+        cmd += ["--become"]
+    if parsed_args.check:
+        cmd += ["--check"]
+    if parsed_args.limit or limit:
+        limits = [l for l in [parsed_args.limit, limit] if l]
+        cmd += ["--limit", "&".join(limits)]
+    if parsed_args.tags or tags:
+        all_tags = [t for t in [parsed_args.tags, tags] if t]
+        cmd += ["--tags", ",".join(all_tags)]
+    cmd += playbooks
+    return cmd
+
+
+def run_playbooks(parsed_args, playbooks,
+                  extra_vars=None, limit=None, tags=None, quiet=False):
+    """Run a Kayobe Ansible playbook."""
+    _validate_args(parsed_args, playbooks)
+    cmd = build_args(parsed_args, playbooks,
+                     extra_vars=extra_vars, limit=limit, tags=tags)
+    try:
+        utils.run_command(cmd, quiet=quiet)
+    except subprocess.CalledProcessError as e:
+        LOG.error("Kayobe playbook(s) %s exited %d",
+                  ", ".join(playbooks), e.returncode)
+        sys.exit(e.returncode)
+
+
+def run_playbook(parsed_args, playbook, *args, **kwargs):
+    """Run a Kayobe Ansible playbook."""
+    return run_playbooks(parsed_args, [playbook], *args, **kwargs)
+
+
+def config_dump(parsed_args, host=None, hosts=None, var_name=None,
+                facts=False, extra_vars=None):
+    dump_dir = tempfile.mkdtemp()
+    try:
+        if not extra_vars:
+            extra_vars = {}
+        extra_vars["dump_path"] = dump_dir
+        if host or hosts:
+            extra_vars["dump_hosts"] = host or hosts
+        if var_name:
+            extra_vars["dump_var_name"] = var_name
+        if facts is not None:
+            extra_vars["dump_facts"] = facts
+        run_playbook(parsed_args, "ansible/dump-config.yml",
+                     extra_vars=extra_vars, quiet=True)
+        hostvars = {}
+        for path in os.listdir(dump_dir):
+            LOG.debug("Found dump file %s", path)
+            inventory_hostname, ext = os.path.splitext(path)
+            if ext == ".yml":
+                hvars = utils.read_yaml_file(os.path.join(dump_dir, path))
+                if host:
+                    return hvars
+                else:
+                    hostvars[inventory_hostname] = hvars
+            else:
+                LOG.warning("Unexpected extension on config dump file %s",
+                            path)
+        return hostvars
+    finally:
+        shutil.rmtree(dump_dir)

kayobe/cli/commands.py

@@ -0,0 +1,212 @@
+import json
+import platform
+import sys
+
+from cliff.command import Command
+
+from kayobe import ansible
+from kayobe import kolla_ansible
+from kayobe import utils
+
+
+class KayobeAnsibleMixin(object):
+    """Mixin class for commands running Kayobe Ansible playbooks."""
+
+    def get_parser(self, prog_name):
+        parser = super(KayobeAnsibleMixin, self).get_parser(prog_name)
+        group = parser.add_argument_group("Kayobe Ansible")
+        ansible.add_args(group)
+        return parser
+
+
+class KollaAnsibleMixin(object):
+    """Mixin class for commands running Kolla Ansible."""
+
+    def get_parser(self, prog_name):
+        parser = super(KollaAnsibleMixin, self).get_parser(prog_name)
+        group = parser.add_argument_group("Kolla Ansible")
+        kolla_ansible.add_args(group)
+        return parser
+
+
+class ControlHostBootstrap(KayobeAnsibleMixin, Command):
+    """Bootstrap the Kayobe control environment."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Bootstrapping Kayobe control host")
+        linux_distname = platform.linux_distribution()[0]
+        if linux_distname == "CentOS Linux":
+            utils.yum_install(["epel-release"])
+        else:
+            # On RHEL, the following should be done to install EPEL:
+            # sudo subscription-manager repos --enable=qci-1.0-for-rhel-7-rpms
+            # if ! yum info epel-release >/dev/null 2>&1 ; then
+            #     sudo yum -y install \
+            #         https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+            # fi
+            self.app.LOG.error("%s is not currently supported", linux_distname)
+            sys.exit(1)
+        utils.yum_install(["ansible"])
+        ansible.galaxy_install("ansible/requirements.yml", "ansible/roles")
+        playbooks = ["ansible/%s.yml" % playbook for playbook in
+                     "bootstrap", "kolla"]
+        ansible.run_playbooks(parsed_args, playbooks)
+
+
+class ConfigurationDump(KayobeAnsibleMixin, Command):
+    """Dump Kayobe configuration."""
+
+    def get_parser(self, prog_name):
+        parser = super(ConfigurationDump, self).get_parser(prog_name)
+        group = parser.add_argument_group("Configuration Dump")
+        group.add_argument("--dump-facts", default=False,
+                           help="whether to gather and dump host facts")
+        group.add_argument("--host",
+                           help="name of a host to dump config for")
+        group.add_argument("--hosts",
+                           help="name of hosts and/or groups to dump config "
+                                "for")
+        group.add_argument("--var-name",
+                           help="name of a variable to dump")
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Dumping Ansible configuration")
+        hostvars = ansible.config_dump(parsed_args,
+                                       host=parsed_args.host,
+                                       hosts=parsed_args.hosts,
+                                       facts=parsed_args.dump_facts,
+                                       var_name=parsed_args.var_name)
+        try:
+            json.dump(hostvars, sys.stdout, sort_keys=True, indent=4)
+        except TypeError as e:
+            self.app.LOG.error("Failed to JSON encode configuration: %s",
+                               repr(e))
+            sys.exit(1)
+
+
+class PlaybookRun(KayobeAnsibleMixin, Command):
+    """Run a Kayobe Ansible playbook."""
+
+    def get_parser(self, prog_name):
+        parser = super(PlaybookRun, self).get_parser(prog_name)
+        group = parser.add_argument_group("Kayobe Ansible")
+        group.add_argument("playbook", nargs="+",
+                           help="name of the playbook(s) to run")
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Running Kayobe playbook(s)")
+        ansible.run_playbooks(parsed_args, parsed_args.playbook)
+
+
+class KollaAnsibleRun(KollaAnsibleMixin, Command):
+    """Run a Kolla Ansible command."""
+
+    def get_parser(self, prog_name):
+        parser = super(KollaAnsibleRun, self).get_parser(prog_name)
+        group = parser.add_argument_group("Kolla Ansible")
+        group.add_argument("--kolla-inventory-filename", default="overcloud",
+                           choices=["seed", "overcloud"],
+                           help="name of the kolla-ansible inventory file, "
+                                "one of seed or overcloud (default "
+                                "overcloud)")
+        group.add_argument("command",
+                           help="name of the kolla-ansible command to run")
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Running Kolla Ansible command")
+        kolla_ansible.run(parsed_args, parsed_args.command,
+                          parsed_args.kolla_inventory_filename)
+
+
+class SeedVMProvision(KollaAnsibleMixin, KayobeAnsibleMixin, Command):
+    """Provision the seed VM."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Provisioning seed VM")
+        ansible.run_playbook(parsed_args, "ansible/seed-vm.yml")
+
+
+class SeedDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, Command):
+    """Deploy the seed node services."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Deploying seed services")
+        self._configure_os(parsed_args)
+        self._deploy_bifrost(parsed_args)
+
+    def _configure_os(self, parsed_args):
+        ansible_user = ansible.config_dump(parsed_args, host="seed",
+                                           var_name="kayobe_ansible_user")
+        playbooks = ["ansible/%s.yml" % playbook for playbook in
+                     "ip-allocation", "ssh-known-host", "kayobe-ansible-user",
+                     "disable-selinux", "network", "ntp"]
+        ansible.run_playbooks(parsed_args, playbooks, limit="seed")
+        kolla_ansible.run_seed(parsed_args, "bootstrap-servers",
+                               extra_vars={"ansible_user": ansible_user})
+        playbooks = ["ansible/%s.yml" % playbook for playbook in
+                     "kolla-host", "docker"]
+        ansible.run_playbooks(parsed_args, playbooks, limit="seed")
+
+    def _deploy_bifrost(self, parsed_args):
+        ansible.run_playbook(parsed_args, "ansible/kolla-bifrost.yml")
+        # FIXME: Do this via configuration.
+        extra_vars = {"kolla_install_type": "source",
+                      "docker_namespace": "stackhpc"}
+        kolla_ansible.run_seed(parsed_args, "deploy-bifrost",
+                               extra_vars=extra_vars)
+
+
+class OvercloudProvision(KollaAnsibleMixin, KayobeAnsibleMixin, Command):
+    """Provision the overcloud."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Provisioning overcloud")
+        self._configure_network(parsed_args)
+        self._configure_bios_and_raid(parsed_args)
+        self._deploy_servers(parsed_args)
+
+    def _configure_network(self, parsed_args):
+        self.app.LOG.debug("TODO: configure overcloud network")
+
+    def _configure_bios_and_raid(self, parsed_args):
+        self.app.LOG.debug("TODO: configure overcloud BIOS and RAID")
+
+    def _deploy_servers(self, parsed_args):
+        self.app.LOG.debug("Deploying overcloud servers via Bifrost")
+        kolla_ansible.run_seed(parsed_args, "deploy-servers")
+
+
+class OvercloudDeploy(KollaAnsibleMixin, KayobeAnsibleMixin, Command):
+    """Deploy the overcloud services."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.debug("Deploying overcloud services")
+        self._configure_os(parsed_args)
+        self._deploy_services(parsed_args)
+
+    def _configure_os(self, parsed_args):
+        ansible_user = ansible.config_dump(parsed_args, host="controllers[0]",
+                                           var_name="kayobe_ansible_user")
+        playbooks = ["ansible/%s.yml" % playbook for playbook in
+                     "ip-allocation", "ssh-known-host", "kayobe-ansible-user",
+                     "disable-selinux", "network", "ntp"]
+        ansible.run_playbooks(parsed_args, playbooks, limit="controllers")
+        kolla_ansible.run_overcloud(parsed_args, "bootstrap-servers",
+                                    extra_vars={"ansible_user": ansible_user})
+        playbooks = ["ansible/%s.yml" % playbook for playbook in
+                     "kolla-host", "docker"]
+        ansible.run_playbooks(parsed_args, playbooks, limit="controllers")
+
+    def _deploy_services(self, parsed_args):
+        playbooks = ["ansible/%s.yml" % playbook for playbook in
+                     "kolla-openstack", "swift-setup"]
+        ansible.run_playbooks(parsed_args, playbooks)
+        for command in ["pull", "prechecks", "deploy"]:
+            kolla_ansible.run_overcloud(parsed_args, command)
+        # FIXME: Fudge to work around incorrect configuration path.
+        extra_vars = {"node_config_directory": parsed_args.config_path}
+        kolla_ansible.run_overcloud(parsed_args, command,
+                                    extra_vars=extra_vars)

kayobe/cmd/kayobe.py

@@ -0,0 +1,35 @@
+import sys
+
+from cliff.app import App
+from cliff.commandmanager import CommandManager
+
+
+class KayobeApp(App):
+
+    def __init__(self):
+        super(KayobeApp, self).__init__(
+            description='Kayobe Command Line Interface (CLI)',
+            version='0.1',
+            command_manager=CommandManager('kayobe.cli'),
+            deferred_help=True,
+            )
+
+    def initialize_app(self, argv):
+        self.LOG.debug('initialize_app')
+
+    def prepare_to_run_command(self, cmd):
+        self.LOG.debug('prepare_to_run_command %s', cmd.__class__.__name__)
+
+    def clean_up(self, cmd, result, err):
+        self.LOG.debug('clean_up %s', cmd.__class__.__name__)
+        if err:
+            self.LOG.debug('got an error: %s', err)
+
+
+def main(argv=sys.argv[1:]):
+    myapp = KayobeApp()
+    return myapp.run(argv)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

kayobe/kolla_ansible.py

@@ -0,0 +1,121 @@
+import logging
+import os
+import os.path
+import subprocess
+import sys
+
+from kayobe import utils
+
+
+DEFAULT_CONFIG_PATH = "/etc/kolla"
+
+CONFIG_PATH_ENV = "KOLLA_CONFIG_PATH"
+
+DEFAULT_VENV_PATH = "ansible/kolla-venv"
+
+VENV_PATH_ENV = "KOLLA_VENV"
+
+LOG = logging.getLogger(__name__)
+
+
+def add_args(parser):
+    """Add arguments required for running Kolla Ansible to a parser."""
+    default_config_path = os.getenv(CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH)
+    default_venv = os.getenv(VENV_PATH_ENV, DEFAULT_VENV_PATH)
+    parser.add_argument("--kolla-config-path", default=default_config_path,
+                        help="path to Kolla configuration. "
+                             "(default=$%s or %s)" %
+                             (CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH))
+    parser.add_argument("--kolla-extra-vars", metavar="EXTRA_VARS",
+                        action="append",
+                        help="set additional variables as key=value or "
+                             "YAML/JSON for Kolla Ansible")
+    parser.add_argument("--kolla-inventory", metavar="INVENTORY",
+                        help="specify inventory host path "
+                             "(default=$%s/inventory or %s/inventory) or "
+                             "comma-separated host list for Kolla Ansible" %
+                             (CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH))
+    parser.add_argument("--kolla-tags", metavar="TAGS", action="append",
+                        help="only run plays and tasks tagged with these "
+                             "values in Kolla Ansible")
+    parser.add_argument("--kolla-venv", metavar="VENV", default=default_venv,
+                        help="path to virtualenv where Kolla Ansible is "
+                             "installed")
+
+
+def _get_inventory_path(parsed_args, inventory_filename):
+    """Return the path to the Kolla inventory."""
+    if parsed_args.kolla_inventory:
+        return parsed_args.kolla_inventory
+    else:
+        return os.path.join(parsed_args.kolla_config_path, "inventory",
+                            inventory_filename)
+
+
+def _validate_args(parsed_args, inventory_filename):
+    """Validate Kayobe Ansible arguments."""
+    result = utils.is_readable_dir(parsed_args.kolla_config_path)
+    if not result["result"]:
+        LOG.error("Kolla configuration path %s is invalid: %s",
+                  parsed_args.kolla_config_path, result["message"])
+        sys.exit(1)
+
+    inventory = _get_inventory_path(parsed_args, inventory_filename)
+    result = utils.is_readable_file(inventory)
+    if not result["result"]:
+        LOG.error("Kolla inventory %s is invalid: %s",
+                  inventory, result["message"])
+        sys.exit(1)
+
+    result = utils.is_readable_dir(parsed_args.kolla_venv)
+    if not result["result"]:
+        LOG.error("Kolla virtualenv %s is invalid: %s",
+                  parsed_args.kolla_venv, result["message"])
+        sys.exit(1)
+
+
+def build_args(parsed_args, command, inventory_filename, extra_vars=None,
+               tags=None):
+    """Build arguments required for running Kolla Ansible."""
+    venv_activate = os.path.join(parsed_args.kolla_venv, "bin", "activate")
+    cmd = ["source", venv_activate, "&&"]
+    cmd = ["kolla-ansible", command]
+    inventory = _get_inventory_path(parsed_args, inventory_filename)
+    cmd += ["--inventory", inventory]
+    cmd += ["--configdir", parsed_args.kolla_config_path]
+    cmd += ["--passwords",
+            os.path.join(parsed_args.kolla_config_path, "passwords.yml")]
+    if parsed_args.kolla_extra_vars:
+        for extra_var in parsed_args.kolla_extra_vars:
+            cmd += ["-e", extra_var]
+    if extra_vars:
+        for extra_var_name, extra_var_value in extra_vars.items():
+            cmd += ["-e", "%s=%s" % (extra_var_name, extra_var_value)]
+    if parsed_args.kolla_tags or tags:
+        all_tags = [t for t in [parsed_args.kolla_tags, tags] if t]
+        cmd += ["--tags", ",".join(all_tags)]
+    return cmd
+
+
+def run(parsed_args, command, inventory_filename, extra_vars=None,
+        tags=None, quiet=False):
+    """Run a Kolla Ansible command."""
+    _validate_args(parsed_args, inventory_filename)
+    cmd = build_args(parsed_args, command,
+                     inventory_filename=inventory_filename,
+                     extra_vars=extra_vars, tags=tags)
+    try:
+        utils.run_command(" ".join(cmd), quiet=quiet, shell=True)
+    except subprocess.CalledProcessError as e:
+        LOG.error("kolla-ansible %s exited %d", command, e.returncode)
+        sys.exit(e.returncode)
+
+
+def run_seed(*args, **kwargs):
+    """Run a Kolla Ansible command using the seed inventory."""
+    return run(*args, inventory_filename="seed", **kwargs)
+
+
+def run_overcloud(*args, **kwargs):
+    """Run a Kolla Ansible command using the overcloud inventory."""
+    return run(*args, inventory_filename="overcloud", **kwargs)

kayobe/utils.py

@@ -0,0 +1,68 @@
+import logging
+import os
+import subprocess
+import sys
+import yaml
+
+
+LOG = logging.getLogger(__name__)
+
+
+def yum_install(packages):
+    """Install a list of packages via Yum."""
+    cmd = ["sudo", "yum", "-y", "install"]
+    cmd += packages
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError as e:
+        print ("Failed to install packages %s via Yum: returncode %d" %
+               (", ".join(packages), e.returncode))
+        sys.exit(e.returncode)
+
+
+def read_yaml_file(path):
+    """Read and decode a YAML file."""
+    try:
+        with open(path, "r") as f:
+            content = f.read()
+    except IOError as e:
+        print ("Failed to open config dump file %s: %s" %
+               (path, repr(e)))
+        sys.exit(1)
+    try:
+        return yaml.load(content)
+    except ValueError as e:
+        print ("Failed to decode config dump YAML file %s: %s" %
+               (path, repr(e)))
+        sys.exit(1)
+
+
+def is_readable_dir(path):
+    """Check whether a path references a readable directory."""
+    if not os.path.exists(path):
+        return {"result": False, "message": "Path does not exist"}
+    if not os.path.isdir(path):
+        return {"result": False, "message": "Path is not a directory"}
+    if not os.access(path, os.R_OK):
+        return {"result": False, "message": "Directory is not readable"}
+    return {"result": True}
+
+
+def is_readable_file(path):
+    """Check whether a path references a readable file."""
+    if not os.path.exists(path):
+        return {"result": False, "message": "Path does not exist"}
+    if not os.path.isfile(path):
+        return {"result": False, "message": "Path is not a file"}
+    if not os.access(path, os.R_OK):
+        return {"result": False, "message": "File is not readable"}
+    return {"result": True}
+
+
+def run_command(cmd, quiet=False, **kwargs):
+    """Run a command, checking the output."""
+    if quiet:
+        kwargs["stdout"] = subprocess.PIPE
+        kwargs["stderr"] = subprocess.PIPE
+    LOG.debug("Running command: %s", " ".join(cmd))
+    subprocess.check_call(cmd, **kwargs)

provision-overcloud.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-
-set -e
-
-function run_kolla_ansible {
-    export KOLLA_CONFIG_PATH=${KOLLA_CONFIG_PATH:-/etc/kolla}
-    # Ansible fails silently if the inventory does not exist.
-    test -e ${KOLLA_CONFIG_PATH}/inventory/seed
-    KOLLA_VENV=$(pwd)/ansible/kolla-venv
-    source ${KOLLA_VENV}/bin/activate
-    kolla-ansible \
-        --configdir ${KOLLA_CONFIG_PATH} \
-        --passwords ${KOLLA_CONFIG_PATH}/passwords.yml \
-        -i ${KOLLA_CONFIG_PATH}/inventory/seed \
-        $@
-    deactivate
-}
-
-function configure_network {
-    echo "TODO: configure overcloud network"
-}
-
-function configure_bios_and_raid {
-    echo "TODO: configure overcloud BIOS and RAID"
-}
-
-function deploy_servers {
-    # Deploy servers with Bifrost
-    run_kolla_ansible deploy-servers
-}
-
-function provision_overcloud {
-    configure_network
-    configure_bios_and_raid
-    deploy_servers
-}
-
-###########################################################
-# Main
-
-function main {
-    provision_overcloud
-}
-
-provision_overcloud

provision-seed.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-set -e
-
-function run_playbook {
-    KAYOBE_CONFIG_PATH=${KAYOBE_CONFIG_PATH:-/etc/kayobe}
-    # Ansible fails silently if the inventory does not exist.
-    test -e ${KAYOBE_CONFIG_PATH}/inventory
-    ansible-playbook \
-        -i ${KAYOBE_CONFIG_PATH}/inventory \
-        -e @${KAYOBE_CONFIG_PATH}/dns.yml \
-        -e @${KAYOBE_CONFIG_PATH}/globals.yml \
-        -e @${KAYOBE_CONFIG_PATH}/kolla.yml \
-        -e @${KAYOBE_CONFIG_PATH}/networks.yml \
-        -e @${KAYOBE_CONFIG_PATH}/network-allocation.yml \
-        -e @${KAYOBE_CONFIG_PATH}/ntp.yml \
-        -e @${KAYOBE_CONFIG_PATH}/seed-vm.yml \
-        -e @${KAYOBE_CONFIG_PATH}/ssh.yml \
-        -e @${KAYOBE_CONFIG_PATH}/swift.yml \
-        $@
-}
-
-function provision_seed_vm {
-    run_playbook ansible/seed-vm.yml
-}
-
-###########################################################
-# Main
-
-function main {
-    provision_seed_vm
-}
-
-main $*

requirements.txt

@@ -0,0 +1 @@
+cliff

setup.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+from setuptools import setup, find_packages
+
+
+PROJECT = 'kayobe'
+VERSION = '0.1'
+
+try:
+    long_description = open('README.md', 'rt').read()
+except IOError:
+    long_description = ''
+
+setup(
+    name=PROJECT,
+    version=VERSION,
+
+    description='OpenStack deployment for scientific computing',
+    long_description=long_description,
+
+    author='StackHPC',
+    author_email='mark@stackhpc.com',
+
+    url='https://github.com/stackhpc/kayobe',
+    download_url='https://github.com/stackhpc/kayobe/tarball/master',
+
+    provides=[],
+    install_requires=['cliff'],
+
+    namespace_packages=[],
+    packages=find_packages(),
+    include_package_data=True,
+
+    entry_points={
+        'console_scripts': [
+            'kayobe = kayobe.cmd.kayobe:main'
+        ],
+        'kayobe.cli': [
+            'control_host_bootstrap = kayobe.cli.commands:ControlHostBootstrap',
+            'configuration_dump = kayobe.cli.commands:ConfigurationDump',
+            'kolla_ansible_run = kayobe.cli.commands:KollaAnsibleRun',
+            'overcloud_deploy = kayobe.cli.commands:OvercloudDeploy',
+            'overcloud_provision = kayobe.cli.commands:OvercloudProvision',
+            'playbook_run = kayobe.cli.commands:PlaybookRun',
+            'seed_deploy = kayobe.cli.commands:SeedDeploy',
+            'seed_vm_provision = kayobe.cli.commands:SeedVMProvision',
+        ],
+    },
+
+    zip_safe=False,
+)