From 12c6f7120faa14a70c089f01c904a1e731c77c66 Mon Sep 17 00:00:00 2001 From: Dmitry Tantsur Date: Thu, 4 Jun 2020 13:07:13 +0200 Subject: [PATCH] Quick start Bifrost CLI This change adds a local CLI ./bifrost-cli that allows to easily run bifrost playbooks. It is targeting early adopters and thus is opinionated and does not expose all possible options. Only the very minimum is provided in this patch. More options will be added later as we decide they are important. Documentation will also be provided later as it's going to be quite large. This feature should be considered technical preview at this point until we give it more testing. Change-Id: I2205e759431024124518716eccd07f79bda14f3a --- .gitignore | 4 + bifrost-cli | 13 +++ bifrost/__init__.py | 10 +- bifrost/cli.py | 234 +++++++++++++++++++++++++++++++++++++++ playbooks/ci/run.yaml | 1 + scripts/test-bifrost.sh | 55 ++++----- zuul.d/bifrost-jobs.yaml | 6 + zuul.d/project.yaml | 2 + 8 files changed, 294 insertions(+), 31 deletions(-) create mode 100755 bifrost-cli create mode 100644 bifrost/cli.py diff --git a/.gitignore b/.gitignore index a2f5dbe67..cafdacd5a 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,7 @@ playbooks/library/os_keystone_service.py # Other *.DS_Store .idea + +# Generated by bifrost-cli +baremetal-inventory.json +baremetal-nodes.json diff --git a/bifrost-cli b/bifrost-cli new file mode 100755 index 000000000..95efe299d --- /dev/null +++ b/bifrost-cli @@ -0,0 +1,13 @@ +#!/bin/sh + +if ! python3 --version > /dev/null; then + echo "Python 3 not found, version 3.6 or newer is required for Bifrost" + exit 1 +fi + +if ! python3 -c "import sys; assert sys.version_info >= (3, 6)" 2> /dev/null; then + echo "Python 3.6 or newer is required for Bifrost" + exit 1 +fi + +PYTHONPATH=$(dirname $0) exec python3 -m bifrost.cli $@ diff --git a/bifrost/__init__.py b/bifrost/__init__.py index 7621e0dbf..3a68a217c 100644 --- a/bifrost/__init__.py +++ b/bifrost/__init__.py @@ -12,11 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -import pbr.version +try: + import pbr.version - -__version__ = pbr.version.VersionInfo( - 'bifrost').version_string() + __version__ = pbr.version.VersionInfo( + 'bifrost').version_string() +except ImportError: + pass # Allow the CLI to work without pbr installed __all__ = [ 'inventory' diff --git a/bifrost/cli.py b/bifrost/cli.py new file mode 100644 index 000000000..3d0e91667 --- /dev/null +++ b/bifrost/cli.py @@ -0,0 +1,234 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import configparser +import ipaddress +import itertools +import os +import subprocess +import sys + + +VENV = "/opt/stack/bifrost" +ANSIBLE = os.path.join(VENV, 'bin', 'ansible-playbook') +COMMON_ENV = { + 'VENV': VENV, + 'USE_VENV': 'true', + 'ENABLE_VENV': 'true', +} +COMMON_PARAMS = [ + '-e', 'ansible_python_interpreter=%s/bin/python3' % VENV, + '-e', 'enable_venv=true', + '-e', 'bifrost_venv_dir=%s' % VENV, +] +BASE = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +PLAYBOOKS = os.path.join(BASE, 'playbooks') + + +def get_env(extra=None): + # NOTE(dtantsur): the order here matters! + result = os.environ.copy() + result.update(COMMON_ENV) + if extra: + result.update(extra) + return result + + +def log(*message, only_if=True): + if only_if: + print(*message, file=sys.stderr) + + +def ansible(playbook, inventory, verbose=False, env=None, **params): + extra = COMMON_PARAMS + list(itertools.chain.from_iterable( + ('-e', '%s=%s' % pair) for pair in params.items() + if pair[1] is not None)) + if verbose: + extra.append('-vvvv') + args = [ANSIBLE, playbook, '-i', inventory] + extra + log('Calling ansible with', args, 'and environment', env, only_if=verbose) + subprocess.check_call(args, env=get_env(env), cwd=PLAYBOOKS) + + +def env_setup(args): + if os.path.exists(VENV): + log(VENV, 'exists, skipping environment preparation', + only_if=args.debug) + return + + log('Installing dependencies and preparing an environment in', VENV) + subprocess.check_call(["bash", "scripts/env-setup.sh"], + env=get_env(), cwd=BASE) + + +def get_release(release): + if release: + if release != 'master' and not release.startswith('stable/'): + release = 'stable/%s' % release + return release + else: + try: + gr = configparser.ConfigParser() + gr.read(os.path.join(BASE, '.gitreview')) + release = gr.get('gerrit', 'defaultbranch', fallback='master') + log('Using release', release, 'detected from the checkout') + return release + except (FileNotFoundError, configparser.Error): + log('Cannot read .gitreview, falling back to release "master"') + return 'master' + + +def cmd_testenv(args): + release = get_release(args.release) + + env_setup(args) + log('Creating', args.count, 'test node(s) with', args.memory, + 'MiB RAM and', args.disk, 'GiB of disk') + + kwargs = {} + if args.storage_pool_path: + kwargs['test_vm_storage_pool_path'] = os.path.abspath( + args.storage_pool_path) + + ansible('test-bifrost-create-vm.yaml', + inventory='inventory/localhost', + verbose=args.debug, + git_branch=release, + test_vm_num_nodes=args.count, + test_vm_memory_size=args.memory, + test_vm_disk_gib=args.disk, + test_vm_domain_type=args.domain_type, + baremetal_json_file=os.path.abspath(args.inventory), + baremetal_nodes_json=os.path.abspath(args.output), + **kwargs) + log('Inventory generated in', args.output) + + +def cmd_install(args): + release = get_release(args.release) + + kwargs = {} + if args.dhcp_pool: + try: + start, end = args.dhcp_pool.split('-') + ipaddress.ip_address(start) + ipaddress.ip_address(end) + except ValueError as e: + sys.exit("Malformed --dhcp-pool, expected two IP addresses. " + "Error: %s" % e) + else: + kwargs['dhcp_pool_start'] = start + kwargs['dhcp_pool_end'] = end + + env_setup(args) + ansible('install.yaml', + inventory='inventory/target', + verbose=args.debug, + git_branch=release, + ipa_upstream_release=release.replace('/', '-'), + create_ipa_image='false', + create_image_via_dib='false', + install_dib='true', + network_interface=args.network_interface, + enable_keystone=args.enable_keystone, + use_public_urls=args.enable_keystone, + noauth_mode=not args.enable_keystone, + testing=args.testenv, + use_cirros=args.testenv, + use_tinyipa=args.testenv, + **kwargs) + log("Ironic is installed and running, try it yourself:\n", + " $ source %s/bin/activate\n" % VENV, + " $ export OS_CLOUD=bifrost\n", + " $ baremetal driver list\n" + "See documentation for next steps") + + +def parse_args(): + parser = argparse.ArgumentParser("Bifrost CLI") + parser.add_argument('--debug', action='store_true', + help='output extensive logging') + + subparsers = parser.add_subparsers() + + testenv = subparsers.add_parser( + 'testenv', help='Prepare a virtual testing environment') + testenv.set_defaults(func=cmd_testenv) + testenv.add_argument('--release', default='master', + help='release branch to use (master, ussuri, etc)') + testenv.add_argument('--count', type=int, default=2, + help='number of nodes to create') + testenv.add_argument('--memory', type=int, default=3072, + help='memory (in MiB) for test nodes') + testenv.add_argument('--disk', type=int, default=10, + help='disk size (in GiB) for test nodes') + testenv.add_argument('--domain-type', default='qemu', + help='domain type: qemu or kvm') + testenv.add_argument('--storage-pool-path', + help='path to libvirt storage pool to setup') + testenv.add_argument('--inventory', default='baremetal-inventory.json', + help='output file with the inventory for using ' + 'with dynamic playbooks') + testenv.add_argument('-o', '--output', default='baremetal-nodes.json', + help='output file with the nodes information for ' + 'importing into ironic') + + install = subparsers.add_parser('install', help='Install ironic') + install.set_defaults(func=cmd_install) + install.add_argument('--testenv', action='store_true', + help='running in a virtual environment') + install.add_argument('--dhcp-pool', metavar='START-END', + help='DHCP pool to use') + install.add_argument('--release', + help='release branch to use (master, ussuri, etc), ' + 'the default value is determined from the ' + '.gitreview file in the source tree') + install.add_argument('--network-interface', + help='the network interface to use') + install.add_argument('--enable-keystone', action='store_true', + help='enable keystone and use authentication') + + args = parser.parse_args() + if getattr(args, 'func', None) is None: + parser.print_usage(file=sys.stderr) + sys.exit("Bifrost CLI: error: a command is required") + return args + + +def check_for_root(): + try: + subprocess.check_call( + '[ $(whoami) == root ] || sudo --non-interactive true', + shell=True, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + # TODO(dtantsur): tell ansible to ask for password + sys.exit('Sudo without password is required for Bifrost') + + +def main(): + args = parse_args() + try: + check_for_root() + args.func(args) + except Exception as exc: + if args.debug: + raise + else: + sys.exit(str(exc)) + except KeyboardInterrupt: + sys.exit('Aborting by user request') + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/playbooks/ci/run.yaml b/playbooks/ci/run.yaml index 0ae150373..e1ec411a7 100644 --- a/playbooks/ci/run.yaml +++ b/playbooks/ci/run.yaml @@ -5,6 +5,7 @@ chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/bifrost'].src_dir }}" environment: BUILD_IMAGE: "{{ build_image | default(false) | bool | lower }}" + CLI_TEST: "{{ cli_test | default(false) | bool | lower }}" ENABLE_KEYSTONE: "{{ enable_keystone | default(false) | bool | lower }}" LOG_LOCATION: "{{ ansible_user_dir }}/logs" UPPER_CONSTRAINTS_FILE: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt" diff --git a/scripts/test-bifrost.sh b/scripts/test-bifrost.sh index 14a1ee51e..4bd611321 100755 --- a/scripts/test-bifrost.sh +++ b/scripts/test-bifrost.sh @@ -11,7 +11,8 @@ BUILD_IMAGE="${BUILD_IMAGE:-false}" BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'} ENABLE_KEYSTONE="${ENABLE_KEYSTONE:-false}" ZUUL_BRANCH=${ZUUL_BRANCH:-} -ENABLE_VENV=${ENABLE_VENV:-true} +ENABLE_VENV=true +CLI_TEST=${CLI_TEST:-false} # Set defaults for ansible command-line options to drive the different # tests. @@ -50,22 +51,18 @@ OS_DISTRO="$ID" # Setup openstack_ci test database if run in OpenStack CI. if [ "$ZUUL_BRANCH" != "" ]; then sudo mkdir -p /opt/libvirt/images - VM_SETUP_EXTRA="-e test_vm_storage_pool_path=/opt/libvirt/images" + VM_SETUP_EXTRA="--storage-pool-path /opt/libvirt/images" fi source $SCRIPT_HOME/env-setup.sh -if [ ${ENABLE_VENV} = "true" ]; then - # Note(cinerama): activate is not compatible with "set -u"; - # disable it just for this line. - set +u - source ${VENV}/bin/activate - set -u - ANSIBLE=${VENV}/bin/ansible-playbook - ANSIBLE_PYTHON_INTERP=${VENV}/bin/python3 -else - ANSIBLE=${HOME}/.local/bin/ansible-playbook - ANSIBLE_PYTHON_INTERP=$(which python3) -fi + +# Note(cinerama): activate is not compatible with "set -u"; +# disable it just for this line. +set +u +source ${VENV}/bin/activate +set -u +ANSIBLE=${VENV}/bin/ansible-playbook +ANSIBLE_PYTHON_INTERP=${VENV}/bin/python3 # Adjust options for DHCP, VM, or Keystone tests if [ ${USE_DHCP} = "true" ]; then @@ -120,19 +117,13 @@ for task in syntax-check list-tasks; do -e testing_user=${TESTING_USER} done -# Create the test VM -${ANSIBLE} -vvvv \ - -i inventory/localhost \ - test-bifrost-create-vm.yaml \ - -e ansible_python_interpreter="${ANSIBLE_PYTHON_INTERP}" \ - -e test_vm_num_nodes=${TEST_VM_NUM_NODES} \ - -e test_vm_memory_size=${VM_MEMORY_SIZE:-512} \ - -e test_vm_domain_type=${VM_DOMAIN_TYPE} \ - -e test_vm_disk_gib=${VM_DISK:-5} \ - -e baremetal_json_file=${BAREMETAL_DATA_FILE} \ - -e enable_venv=${ENABLE_VENV} \ - -e bifrost_venv_dir=${VENV} \ - ${VM_SETUP_EXTRA:-} +# Create the test VMs +../bifrost-cli --debug testenv \ + --count ${TEST_VM_NUM_NODES} \ + --memory ${VM_MEMORY_SIZE:-512} \ + --disk ${VM_DISK:-5} \ + --inventory "${BAREMETAL_DATA_FILE}" \ + ${VM_SETUP_EXTRA:-} if [ ${USE_DHCP} = "true" ]; then # reduce the number of nodes in JSON file @@ -144,6 +135,16 @@ if [ ${USE_DHCP} = "true" ]; then && mv ${BAREMETAL_DATA_FILE}.new ${BAREMETAL_DATA_FILE} fi +if [ ${CLI_TEST} = "true" ]; then + # FIXME(dtantsur): bifrost-cli does not use opendev-provided repos. + ../bifrost-cli --debug install --release ${ZUUL_BRANCH:-master} --testenv + CLOUD_CONFIG+=" -e skip_install=true" + CLOUD_CONFIG+=" -e skip_package_install=true" + CLOUD_CONFIG+=" -e skip_bootstrap=true" + CLOUD_CONFIG+=" -e skip_start=true" + CLOUD_CONFIG+=" -e skip_migrations=true" +fi + set +e # Set BIFROST_INVENTORY_SOURCE diff --git a/zuul.d/bifrost-jobs.yaml b/zuul.d/bifrost-jobs.yaml index d3479b0df..f5f1547a4 100644 --- a/zuul.d/bifrost-jobs.yaml +++ b/zuul.d/bifrost-jobs.yaml @@ -100,6 +100,12 @@ vars: enable_keystone: true +- job: + name: bifrost-cli-ubuntu-bionic + parent: bifrost-integration-tinyipa-ubuntu-bionic + vars: + cli_test: true + - job: name: bifrost-integration-tinyipa-ubuntu-focal parent: bifrost-integration-tinyipa diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 1127fd1c6..026cabc3f 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -34,6 +34,8 @@ voting: false - bifrost-integration-dhcp-debian-buster: voting: false + - bifrost-cli-ubuntu-bionic: + voting: false gate: jobs: - bifrost-integration-tinyipa-ubuntu-bionic