Quick start Bifrost CLI

This change adds a local CLI ./bifrost-cli that allows to easily
run bifrost playbooks. It is targeting early adopters and thus
is opinionated and does not expose all possible options.

Only the very minimum is provided in this patch. More options
will be added later as we decide they are important.

Documentation will also be provided later as it's going to be
quite large. This feature should be considered technical preview
at this point until we give it more testing.

Change-Id: I2205e759431024124518716eccd07f79bda14f3a

.gitignore

@@ -71,3 +71,7 @@ playbooks/library/os_keystone_service.py
 # Other
 *.DS_Store
 .idea
+
+# Generated by bifrost-cli
+baremetal-inventory.json
+baremetal-nodes.json

bifrost-cli

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+if ! python3 --version > /dev/null; then
+    echo "Python 3 not found, version 3.6 or newer is required for Bifrost"
+    exit 1
+fi
+
+if ! python3 -c "import sys; assert sys.version_info >= (3, 6)" 2> /dev/null; then
+    echo "Python 3.6 or newer is required for Bifrost"
+    exit 1
+fi
+
+PYTHONPATH=$(dirname $0) exec python3 -m bifrost.cli $@

bifrost/__init__.py

@@ -12,11 +12,13 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+try:
     import pbr.version
 
-
     __version__ = pbr.version.VersionInfo(
         'bifrost').version_string()
+except ImportError:
+    pass  # Allow the CLI to work without pbr installed
 
 __all__ = [
     'inventory'

bifrost/cli.py

@@ -0,0 +1,234 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import configparser
+import ipaddress
+import itertools
+import os
+import subprocess
+import sys
+
+
+VENV = "/opt/stack/bifrost"
+ANSIBLE = os.path.join(VENV, 'bin', 'ansible-playbook')
+COMMON_ENV = {
+    'VENV': VENV,
+    'USE_VENV': 'true',
+    'ENABLE_VENV': 'true',
+}
+COMMON_PARAMS = [
+    '-e', 'ansible_python_interpreter=%s/bin/python3' % VENV,
+    '-e', 'enable_venv=true',
+    '-e', 'bifrost_venv_dir=%s' % VENV,
+]
+BASE = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+PLAYBOOKS = os.path.join(BASE, 'playbooks')
+
+
+def get_env(extra=None):
+    # NOTE(dtantsur): the order here matters!
+    result = os.environ.copy()
+    result.update(COMMON_ENV)
+    if extra:
+        result.update(extra)
+    return result
+
+
+def log(*message, only_if=True):
+    if only_if:
+        print(*message, file=sys.stderr)
+
+
+def ansible(playbook, inventory, verbose=False, env=None, **params):
+    extra = COMMON_PARAMS + list(itertools.chain.from_iterable(
+        ('-e', '%s=%s' % pair) for pair in params.items()
+        if pair[1] is not None))
+    if verbose:
+        extra.append('-vvvv')
+    args = [ANSIBLE, playbook, '-i', inventory] + extra
+    log('Calling ansible with', args, 'and environment', env, only_if=verbose)
+    subprocess.check_call(args, env=get_env(env), cwd=PLAYBOOKS)
+
+
+def env_setup(args):
+    if os.path.exists(VENV):
+        log(VENV, 'exists, skipping environment preparation',
+            only_if=args.debug)
+        return
+
+    log('Installing dependencies and preparing an environment in', VENV)
+    subprocess.check_call(["bash", "scripts/env-setup.sh"],
+                          env=get_env(), cwd=BASE)
+
+
+def get_release(release):
+    if release:
+        if release != 'master' and not release.startswith('stable/'):
+            release = 'stable/%s' % release
+        return release
+    else:
+        try:
+            gr = configparser.ConfigParser()
+            gr.read(os.path.join(BASE, '.gitreview'))
+            release = gr.get('gerrit', 'defaultbranch', fallback='master')
+            log('Using release', release, 'detected from the checkout')
+            return release
+        except (FileNotFoundError, configparser.Error):
+            log('Cannot read .gitreview, falling back to release "master"')
+            return 'master'
+
+
+def cmd_testenv(args):
+    release = get_release(args.release)
+
+    env_setup(args)
+    log('Creating', args.count, 'test node(s) with', args.memory,
+        'MiB RAM and', args.disk, 'GiB of disk')
+
+    kwargs = {}
+    if args.storage_pool_path:
+        kwargs['test_vm_storage_pool_path'] = os.path.abspath(
+            args.storage_pool_path)
+
+    ansible('test-bifrost-create-vm.yaml',
+            inventory='inventory/localhost',
+            verbose=args.debug,
+            git_branch=release,
+            test_vm_num_nodes=args.count,
+            test_vm_memory_size=args.memory,
+            test_vm_disk_gib=args.disk,
+            test_vm_domain_type=args.domain_type,
+            baremetal_json_file=os.path.abspath(args.inventory),
+            baremetal_nodes_json=os.path.abspath(args.output),
+            **kwargs)
+    log('Inventory generated in', args.output)
+
+
+def cmd_install(args):
+    release = get_release(args.release)
+
+    kwargs = {}
+    if args.dhcp_pool:
+        try:
+            start, end = args.dhcp_pool.split('-')
+            ipaddress.ip_address(start)
+            ipaddress.ip_address(end)
+        except ValueError as e:
+            sys.exit("Malformed --dhcp-pool, expected two IP addresses. "
+                     "Error: %s" % e)
+        else:
+            kwargs['dhcp_pool_start'] = start
+            kwargs['dhcp_pool_end'] = end
+
+    env_setup(args)
+    ansible('install.yaml',
+            inventory='inventory/target',
+            verbose=args.debug,
+            git_branch=release,
+            ipa_upstream_release=release.replace('/', '-'),
+            create_ipa_image='false',
+            create_image_via_dib='false',
+            install_dib='true',
+            network_interface=args.network_interface,
+            enable_keystone=args.enable_keystone,
+            use_public_urls=args.enable_keystone,
+            noauth_mode=not args.enable_keystone,
+            testing=args.testenv,
+            use_cirros=args.testenv,
+            use_tinyipa=args.testenv,
+            **kwargs)
+    log("Ironic is installed and running, try it yourself:\n",
+        " $ source %s/bin/activate\n" % VENV,
+        " $ export OS_CLOUD=bifrost\n",
+        " $ baremetal driver list\n"
+        "See documentation for next steps")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser("Bifrost CLI")
+    parser.add_argument('--debug', action='store_true',
+                        help='output extensive logging')
+
+    subparsers = parser.add_subparsers()
+
+    testenv = subparsers.add_parser(
+        'testenv', help='Prepare a virtual testing environment')
+    testenv.set_defaults(func=cmd_testenv)
+    testenv.add_argument('--release', default='master',
+                         help='release branch to use (master, ussuri, etc)')
+    testenv.add_argument('--count', type=int, default=2,
+                         help='number of nodes to create')
+    testenv.add_argument('--memory', type=int, default=3072,
+                         help='memory (in MiB) for test nodes')
+    testenv.add_argument('--disk', type=int, default=10,
+                         help='disk size (in GiB) for test nodes')
+    testenv.add_argument('--domain-type', default='qemu',
+                         help='domain type: qemu or kvm')
+    testenv.add_argument('--storage-pool-path',
+                         help='path to libvirt storage pool to setup')
+    testenv.add_argument('--inventory', default='baremetal-inventory.json',
+                         help='output file with the inventory for using '
+                              'with dynamic playbooks')
+    testenv.add_argument('-o', '--output', default='baremetal-nodes.json',
+                         help='output file with the nodes information for '
+                              'importing into ironic')
+
+    install = subparsers.add_parser('install', help='Install ironic')
+    install.set_defaults(func=cmd_install)
+    install.add_argument('--testenv', action='store_true',
+                         help='running in a virtual environment')
+    install.add_argument('--dhcp-pool', metavar='START-END',
+                         help='DHCP pool to use')
+    install.add_argument('--release',
+                         help='release branch to use (master, ussuri, etc), '
+                              'the default value is determined from the '
+                              '.gitreview file in the source tree')
+    install.add_argument('--network-interface',
+                         help='the network interface to use')
+    install.add_argument('--enable-keystone', action='store_true',
+                         help='enable keystone and use authentication')
+
+    args = parser.parse_args()
+    if getattr(args, 'func', None) is None:
+        parser.print_usage(file=sys.stderr)
+        sys.exit("Bifrost CLI: error: a command is required")
+    return args
+
+
+def check_for_root():
+    try:
+        subprocess.check_call(
+            '[ $(whoami) == root ] || sudo --non-interactive true',
+            shell=True, stderr=subprocess.DEVNULL)
+    except subprocess.CalledProcessError:
+        # TODO(dtantsur): tell ansible to ask for password
+        sys.exit('Sudo without password is required for Bifrost')
+
+
+def main():
+    args = parse_args()
+    try:
+        check_for_root()
+        args.func(args)
+    except Exception as exc:
+        if args.debug:
+            raise
+        else:
+            sys.exit(str(exc))
+    except KeyboardInterrupt:
+        sys.exit('Aborting by user request')
+
+
+if __name__ == '__main__':
+    sys.exit(main())

playbooks/ci/run.yaml

@@ -5,6 +5,7 @@
         chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/bifrost'].src_dir }}"
       environment:
         BUILD_IMAGE: "{{ build_image | default(false) | bool | lower }}"
+        CLI_TEST: "{{ cli_test | default(false) | bool | lower }}"
         ENABLE_KEYSTONE: "{{ enable_keystone | default(false) | bool | lower }}"
         LOG_LOCATION: "{{ ansible_user_dir }}/logs"
         UPPER_CONSTRAINTS_FILE: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt"

scripts/test-bifrost.sh

@@ -11,7 +11,8 @@ BUILD_IMAGE="${BUILD_IMAGE:-false}"
 BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'}
 ENABLE_KEYSTONE="${ENABLE_KEYSTONE:-false}"
 ZUUL_BRANCH=${ZUUL_BRANCH:-}
-ENABLE_VENV=${ENABLE_VENV:-true}
+ENABLE_VENV=true
+CLI_TEST=${CLI_TEST:-false}
 
 # Set defaults for ansible command-line options to drive the different
 # tests.
@@ -50,11 +51,11 @@ OS_DISTRO="$ID"
 # Setup openstack_ci test database if run in OpenStack CI.
 if [ "$ZUUL_BRANCH" != "" ]; then
     sudo mkdir -p /opt/libvirt/images
-    VM_SETUP_EXTRA="-e test_vm_storage_pool_path=/opt/libvirt/images"
+    VM_SETUP_EXTRA="--storage-pool-path /opt/libvirt/images"
 fi
 
 source $SCRIPT_HOME/env-setup.sh
-if [ ${ENABLE_VENV} = "true" ]; then
+
 # Note(cinerama): activate is not compatible with "set -u";
 # disable it just for this line.
 set +u
@@ -62,10 +63,6 @@ if [ ${ENABLE_VENV} = "true" ]; then
 set -u
 ANSIBLE=${VENV}/bin/ansible-playbook
 ANSIBLE_PYTHON_INTERP=${VENV}/bin/python3
-else
-    ANSIBLE=${HOME}/.local/bin/ansible-playbook
-    ANSIBLE_PYTHON_INTERP=$(which python3)
-fi
 
 # Adjust options for DHCP, VM, or Keystone tests
 if [ ${USE_DHCP} = "true" ]; then
@@ -120,18 +117,12 @@ for task in syntax-check list-tasks; do
            -e testing_user=${TESTING_USER}
 done
 
-# Create the test VM
-${ANSIBLE} -vvvv \
-       -i inventory/localhost \
-       test-bifrost-create-vm.yaml \
-       -e ansible_python_interpreter="${ANSIBLE_PYTHON_INTERP}" \
-       -e test_vm_num_nodes=${TEST_VM_NUM_NODES} \
-       -e test_vm_memory_size=${VM_MEMORY_SIZE:-512} \
-       -e test_vm_domain_type=${VM_DOMAIN_TYPE} \
-       -e test_vm_disk_gib=${VM_DISK:-5} \
-       -e baremetal_json_file=${BAREMETAL_DATA_FILE} \
-       -e enable_venv=${ENABLE_VENV} \
-       -e bifrost_venv_dir=${VENV} \
+# Create the test VMs
+../bifrost-cli --debug testenv \
+    --count ${TEST_VM_NUM_NODES} \
+    --memory ${VM_MEMORY_SIZE:-512} \
+    --disk ${VM_DISK:-5} \
+    --inventory "${BAREMETAL_DATA_FILE}" \
     ${VM_SETUP_EXTRA:-}
 
 if [ ${USE_DHCP} = "true" ]; then
@@ -144,6 +135,16 @@ if [ ${USE_DHCP} = "true" ]; then
         && mv ${BAREMETAL_DATA_FILE}.new ${BAREMETAL_DATA_FILE}
 fi
 
+if [ ${CLI_TEST} = "true" ]; then
+    # FIXME(dtantsur): bifrost-cli does not use opendev-provided repos.
+    ../bifrost-cli --debug install --release ${ZUUL_BRANCH:-master} --testenv
+    CLOUD_CONFIG+=" -e skip_install=true"
+    CLOUD_CONFIG+=" -e skip_package_install=true"
+    CLOUD_CONFIG+=" -e skip_bootstrap=true"
+    CLOUD_CONFIG+=" -e skip_start=true"
+    CLOUD_CONFIG+=" -e skip_migrations=true"
+fi
+
 set +e
 
 # Set BIFROST_INVENTORY_SOURCE

zuul.d/bifrost-jobs.yaml

@@ -100,6 +100,12 @@
     vars:
       enable_keystone: true
 
+- job:
+    name: bifrost-cli-ubuntu-bionic
+    parent: bifrost-integration-tinyipa-ubuntu-bionic
+    vars:
+      cli_test: true
+
 - job:
     name: bifrost-integration-tinyipa-ubuntu-focal
     parent: bifrost-integration-tinyipa

zuul.d/project.yaml

@@ -34,6 +34,8 @@
            voting: false
         - bifrost-integration-dhcp-debian-buster:
            voting: false
+        - bifrost-cli-ubuntu-bionic:
+           voting: false
     gate:
       jobs:
         - bifrost-integration-tinyipa-ubuntu-bionic