From b3880aa264fdbca4525eb00b77819772c7cda748 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Sat, 21 Oct 2017 09:33:34 +0100 Subject: [PATCH] Add scripting around the development environment The following top level scripts are added under dev/: * overcloud-deploy.sh * seed-deploy.sh * seed-hypervisor-deploy.sh Some amount of configuration is possible via dev/config.sh. --- Vagrantfile | 12 +- dev/config.sh | 19 ++ dev/environment-setup.sh | 20 ++ dev/functions | 195 ++++++++++++++++++ dev/install.sh | 21 ++ dev/overcloud-deploy.sh | 19 ++ dev/seed-deploy.sh | 18 ++ dev/seed-hypervisor-deploy.sh | 18 ++ doc/source/development/automated.rst | 151 ++++++++++++++ doc/source/{ => development}/contributing.rst | 2 +- doc/source/development/index.rst | 11 + .../manual.rst} | 48 ++--- doc/source/development/vagrant.rst | 55 +++++ doc/source/index.rst | 3 +- 14 files changed, 548 insertions(+), 44 deletions(-) create mode 100644 dev/config.sh create mode 100755 dev/environment-setup.sh create mode 100644 dev/functions create mode 100755 dev/install.sh create mode 100755 dev/overcloud-deploy.sh create mode 100755 dev/seed-deploy.sh create mode 100755 dev/seed-hypervisor-deploy.sh create mode 100644 doc/source/development/automated.rst rename doc/source/{ => development}/contributing.rst (58%) create mode 100644 doc/source/development/index.rst rename doc/source/{development.rst => development/manual.rst} (60%) create mode 100644 doc/source/development/vagrant.rst diff --git a/Vagrantfile b/Vagrantfile index e060a79c3..85a106464 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -43,22 +43,16 @@ NM_CONTROLLED=no EOF sudo ifup eth1 - sudo yum -y install gcc git vim python-virtualenv + /vagrant/dev/install.sh + # Configure the legacy development environment. This has been retained + # while transitioning to the new development environment. cat > /vagrant/kayobe-env << EOF export KAYOBE_CONFIG_PATH=/vagrant/etc/kayobe export KOLLA_CONFIG_PATH=/vagrant/etc/kolla EOF - source /vagrant/kayobe-env - cp /vagrant/dev/dev-vagrant.yml /vagrant/etc/kayobe/ cp /vagrant/dev/dev-hosts /vagrant/etc/kayobe/inventory cp /vagrant/dev/dev-vagrant-network-allocation.yml /vagrant/etc/kayobe/network-allocation.yml - - virtualenv ~/kayobe-venv - source ~/kayobe-venv/bin/activate - pip install -U pip - pip install /vagrant - deactivate SHELL end diff --git a/dev/config.sh b/dev/config.sh new file mode 100644 index 000000000..1d4efcad9 --- /dev/null +++ b/dev/config.sh @@ -0,0 +1,19 @@ +# Configuration for kayobe development environment. + +# Path to the kayobe source code repository. Typically this will be the Vagrant +# shared directory. +#export KAYOBE_SOURCE_PATH=/vagrant + +# Path to the kayobe-config repository checkout. +#export KAYOBE_CONFIG_SOURCE_PATH=${KAYOBE_SOURCE_PATH}/config/src/kayobe-config + +# Path to the kayobe virtual environment. +#export KAYOBE_VENV_PATH=~/kayobe-venv + +# Whether to build container images for the seed services. If 0, they will be +# pulled. +#export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=0 + +# Whether to build container images for the overcloud services. If 0, they will +# be pulled. +#export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=0 diff --git a/dev/environment-setup.sh b/dev/environment-setup.sh new file mode 100755 index 000000000..bfcf0a871 --- /dev/null +++ b/dev/environment-setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +# This script can be used to prepare the environment for use with kayobe. This +# includes setting environment variables and activating the python virtual +# environment. This script should be sourced rather than executed in a +# subprocess. e.g. source dev/environment-setup.sh + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source ${PARENT}/functions + + +function main { + config_init + environment_setup +} + +main diff --git a/dev/functions b/dev/functions new file mode 100644 index 000000000..32578ee5d --- /dev/null +++ b/dev/functions @@ -0,0 +1,195 @@ +#!/bin/bash + +set -e + +# Library of functions for the kayobe development environment. + +# Configuration + +function config_defaults { + # Set default values for kayobe development configuration. + + # Try to detect if we are running in a vagrant VM. + if [[ -e /vagrant ]]; then + KAYOBE_SOURCE_PATH_DEFAULT=/vagrant + else + KAYOBE_SOURCE_PATH_DEFAULT=$(pwd) + fi + + # Path to the kayobe source code repository. Typically this will be the + # Vagrant shared directory. + export KAYOBE_SOURCE_PATH=${KAYOBE_SOURCE_PATH:-$KAYOBE_SOURCE_PATH_DEFAULT} + + # Path to the kayobe-config repository checkout. + export KAYOBE_CONFIG_SOURCE_PATH=${KAYOBE_CONFIG_SOURCE_PATH:-${KAYOBE_SOURCE_PATH}/config/src/kayobe-config} + + # Path to the kayobe virtual environment. + export KAYOBE_VENV_PATH=${KAYOBE_VENV_PATH:-~/kayobe-venv} + + # Whether to build container images for the seed services. If 0, they will + # be pulled. + export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0} + + # Whether to build container images for the overcloud services. If 0, they + # will be pulled. + export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0} +} + +function config_set { + # Source the configuration file, config.sh + + PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + + source ${PARENT}/config.sh +} + +function config_check { + # Check the configuration environment variables. + + if [[ ! -e $KAYOBE_CONFIG_SOURCE_PATH ]]; then + if [[ ${KAYOBE_CONFIG_REQUIRED:-1} -eq 1 ]]; then + echo "Kayobe configuration path $KAYOBE_CONFIG_SOURCE_PATH does not exist" + return 1 + fi + fi + + if [[ ! -e $KAYOBE_SOURCE_PATH ]]; then + echo "Kayobe source path $KAYOBE_SOURCE_PATH does not exist" + return 1 + fi +} + +function config_init { + config_defaults + config_set + config_check +} + +# Installation + +function install_dependencies { + echo "Installing package dependencies for kayobe" + if [[ -e /etc/centos-release ]]; then + sudo yum -y install gcc git vim python-virtualenv + else + sudo apt install -y python-dev python-virtualenv gcc git + fi +} + +function install_venv { + local venv_parent=$(dirname ${KAYOBE_VENV_PATH}) + if [[ ! -d $venv_parent ]]; then + mkdir -p $venv_parent + fi + if [[ ! -f ${KAYOBE_VENV_PATH}/bin/activate ]]; then + echo "Creating kayobe virtual environment in ${KAYOBE_VENV_PATH}" + virtualenv ${KAYOBE_VENV_PATH} + source ${KAYOBE_VENV_PATH}/bin/activate + pip install -U pip + pip install ${KAYOBE_SOURCE_PATH} + deactivate + else + echo "Using existing kayobe virtual environment in ${KAYOBE_VENV_PATH}" + fi +} + +# Deployment + +function is_deploy_image_built_locally { + ipa_build_images=$(kayobe configuration dump --host controllers[0] --var-name ipa_build_images) + [[ $ipa_build_images =~ ^true$ ]] +} + +function environment_setup { + source ${KAYOBE_VENV_PATH}/bin/activate + source ${KAYOBE_CONFIG_SOURCE_PATH}/kayobe-env + + cd ${KAYOBE_SOURCE_PATH} +} + +function seed_hypervisor_deploy { + # Deploy a seed hypervisor. + environment_setup + + echo "Bootstrapping the ansible control host" + kayobe control host bootstrap + + echo "Configuring the seed hypervisor" + kayobe seed hypervisor host configure +} + +function seed_deploy { + # Deploy a kayobe seed in a VM. + environment_setup + + echo "Bootstrapping the ansible control host" + kayobe control host bootstrap + + echo "Provisioning the seed VM" + kayobe seed vm provision + + echo "Configuring the seed host" + kayobe seed host configure + + # Note: This must currently be before host configure, because host + # configure runs kolla-ansible.yml, which validates the presence of the + # built deploy images. + if is_deploy_image_built_locally; then + echo "Building seed deployment images" + kayobe seed deployment image build + else + echo "Not building seed deployment images" + fi + + if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then + echo "Building seed container images" + kayobe seed container image build + else + echo "Not pulling seed container images - no such command yet" + #kayobe seed container image pull + fi + + echo "Deploying containerised seed services" + kayobe seed service deploy +} + +function overcloud_deploy { + # Deploy a kayobe control plane. + echo "Deploying a kayobe development environment. This consists of a " + echo "single node OpenStack control plane." + + environment_setup + + echo "Bootstrapping the ansible control host" + kayobe control host bootstrap + + echo "Configuring the controller host" + kayobe overcloud host configure + + # Note: This must currently be before host configure, because host + # configure runs kolla-ansible.yml, which validates the presence of the + # built deploy images. + if is_deploy_image_built_locally; then + echo "Building overcloud deployment images" + kayobe overcloud deployment image build + else + echo "Not building overcloud deployment images" + fi + + if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then + echo "Building overcloud container images" + kayobe overcloud container image build + else + echo "Pulling overcloud container images" + kayobe overcloud container image pull + fi + + echo "Deploying containerised overcloud services" + kayobe overcloud service deploy + + echo "Performing post-deployment configuration" + source ${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh + kayobe overcloud post configure + + echo "Control plane deployment complete" +} diff --git a/dev/install.sh b/dev/install.sh new file mode 100755 index 000000000..18c2ed3d8 --- /dev/null +++ b/dev/install.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Install kayobe and its dependencies in a virtual environment. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source ${PARENT}/functions + + +function main { + # Don't require kayobe configuration to exist for installation - it is not + # required for the legacy manual deployment procedure. + KAYOBE_CONFIG_REQUIRED=0 + config_init + install_dependencies + install_venv +} + +main diff --git a/dev/overcloud-deploy.sh b/dev/overcloud-deploy.sh new file mode 100755 index 000000000..5f512ab38 --- /dev/null +++ b/dev/overcloud-deploy.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Simple script to stand up a development environment for an OpenStack +# controller in a Vagrant VM using kayobe. This should be executed from within +# the VM. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source ${PARENT}/functions + + +function main { + config_init + overcloud_deploy +} + +main diff --git a/dev/seed-deploy.sh b/dev/seed-deploy.sh new file mode 100755 index 000000000..4730d0568 --- /dev/null +++ b/dev/seed-deploy.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +# Simple script to stand up a development environment for a seed VM using +# kayobe. This should be executed from the hypervisor. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source ${PARENT}/functions + + +function main { + config_init + seed_deploy +} + +main diff --git a/dev/seed-hypervisor-deploy.sh b/dev/seed-hypervisor-deploy.sh new file mode 100755 index 000000000..0d0aeaa30 --- /dev/null +++ b/dev/seed-hypervisor-deploy.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +# Simple script to stand up a development environment for a seed hypervisor +# using kayobe. This should be executed from the hypervisor. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source ${PARENT}/functions + + +function main { + config_init + seed_hypervisor_deploy +} + +main diff --git a/doc/source/development/automated.rst b/doc/source/development/automated.rst new file mode 100644 index 000000000..785b9bf73 --- /dev/null +++ b/doc/source/development/automated.rst @@ -0,0 +1,151 @@ +.. _development-automated: + +=============== +Automated Setup +=============== + +This section provides information on the development tools provided by kayobe +to automate the deployment of various development environments. + +For a manual procedure, see :ref:`development-manual`. + +Overview +======== + +The kayobe development environment automation tooling is built using simple +shell scripts. Some minimal configuration can be applied by setting the +environment variables in `dev/config.sh`. Control plane configuration is +typically provided via the `dev-kayobe-config +`_ repository, although it is +also possible to use your own kayobe configuration. This allows us to build a +development environment that is as close to production as possible. + +Environments +============ + +The following development environments are supported: + +* Overcloud (single OpenStack controller) +* Seed hypervisor +* Seed VM + +The seed VM environment may be used in an environment already deployed as a +seed hypervisor. + +Overcloud +========= + +Preparation +----------- + +Clone the kayobe repository:: + + git clone https://github.com/stackhpc/kayobe + +Change the current directory to the kayobe repository:: + + cd kayobe + +Clone the ``dev-kayobe-config`` repository to ``config/src/kayobe-config``:: + + mkdir -p config/src + git clone https://github.com/stackhpc/dev-kayobe-config config/src/kayobe-config + +Follow the steps in :ref:`development-vagrant` to prepare your environment for +use with Vagrant and bring up a Vagrant VM. + +Inspect the kayobe configuration and make any changes necessary for your +environment. + +Usage +----- + +SSH into the Vagrant VM:: + + vagrant ssh + +Run the ``dev/overcloud-deploy.sh`` script to deploy the OpenStack control +plane:: + + /vagrant/kayobe/dev/overcloud-deploy.sh + +Upon successful completion of this script, the control plane will be active. + +Seed Hypervisor +=============== + +The seed hypervisor development environment is supported for CentOS 7. The +system must be either bare metal, or a VM on a system with nested +virtualisation enabled. + +Preparation +----------- + +The following commands should be executed on the seed hypervisor. + +Clone the kayobe repository:: + + git clone https://github.com/stackhpc/kayobe + +Change the current directory to the kayobe repository:: + + cd kayobe + +Clone the ``add-seed-and-hv`` branch of the ``dev-kayobe-config`` repository to +``config/src/kayobe-config``:: + + mkdir -p config/src + git clone https://github.com/stackhpc/dev-kayobe-config -b add-seed-and-hv config/src/kayobe-config + +Inspect the kayobe configuration and make any changes necessary for your +environment. + +Usage +----- + +Run the ``dev/seed-hypervisor-deploy.sh`` script to deploy the seed +hypervisor:: + + ./dev/seed-hypervisor-deploy.sh + +Upon successful completion of this script, the seed hypervisor will be active. + +Seed VM +======= + +The seed VM should be deployed on a system configured as a libvirt/KVM +hypervisor, using the kayobe seed hypervisor support or otherwise. + +Preparation +----------- + +The following commands should be executed on the seed hypervisor. + +Change the current directory to the kayobe repository:: + + git clone https://github.com/stackhpc/kayobe + +Change to the ``kayobe`` directory:: + + cd kayobe + +Clone the ``add-seed-and-hv`` branch of the ``dev-kayobe-config`` repository to +``config/src/kayobe-config``:: + + mkdir -p config/src + git clone https://github.com/stackhpc/dev-kayobe-config -b add-seed-and-hv config/src/kayobe-config + +Inspect the kayobe configuration and make any changes necessary for your +environment. + +Usage +===== + +Run the ``dev/seed-deploy.sh`` script to deploy the seed VM:: + + ./dev/seed-deploy.sh + +Upon successful completion of this script, the seed VM will be active. The +seed VM may be accessed via SSH as the ``stack`` user:: + + ssh stack@192.168.33.5 diff --git a/doc/source/contributing.rst b/doc/source/development/contributing.rst similarity index 58% rename from doc/source/contributing.rst rename to doc/source/development/contributing.rst index eb44b0c49..7086829a1 100644 --- a/doc/source/contributing.rst +++ b/doc/source/development/contributing.rst @@ -1,4 +1,4 @@ ================= How to Contribute ================= -.. include:: ../../CONTRIBUTING.rst +.. include:: ../../../CONTRIBUTING.rst diff --git a/doc/source/development/index.rst b/doc/source/development/index.rst new file mode 100644 index 000000000..26b748398 --- /dev/null +++ b/doc/source/development/index.rst @@ -0,0 +1,11 @@ +======================== +Kayobe Development Guide +======================== + +.. toctree:: + :maxdepth: 2 + + vagrant + manual + automated + contributing diff --git a/doc/source/development.rst b/doc/source/development/manual.rst similarity index 60% rename from doc/source/development.rst rename to doc/source/development/manual.rst index 1dd6f5021..8718654ab 100644 --- a/doc/source/development.rst +++ b/doc/source/development/manual.rst @@ -1,42 +1,26 @@ -=========== -Development -=========== +.. _development-manual: -This section describes how to set up an OpenStack controller in a virtual -machine using `Vagrant `_ and Kayobe. +============ +Manual Setup +============ + +This section provides a set of manual steps to set up a development environment +for an OpenStack controller in a virtual machine using `Vagrant +`_ and Kayobe. + +For a more automated and flexible procedure, see :ref:`development-automated`. Preparation =========== -First, ensure that Vagrant is installed and correctly configured to use -virtual box. Also install the following vagrant plugins: +Follow the steps in :ref:`development-vagrant` to prepare your environment for +use with Vagrant and bring up a Vagrant VM. - vagrant plugin install vagrant-vbguest - vagrant plugin install vagrant-reload +Manual Installation +=================== -Note: if using Ubuntu 16.04 LTS, you may be unable to install any plugins. To -work around this install the upstream version from www.virtualbox.org. - -Next, clone kayobe:: - - git clone https://github.com/stackhpc/kayobe - -Change the current directory to the kayobe repository:: - - cd kayobe - -Inspect kayobe's ``Vagrantfile``, noting the provisioning steps:: - - less Vagrantfile - -Bring up a virtual machine:: - - vagrant up - -Wait for the VM to boot. - -Installation -============ +Sometimes the best way to learn a tool is to ditch the scripts and perform a +manual installation. SSH into the controller VM:: diff --git a/doc/source/development/vagrant.rst b/doc/source/development/vagrant.rst new file mode 100644 index 000000000..6bd0b16a2 --- /dev/null +++ b/doc/source/development/vagrant.rst @@ -0,0 +1,55 @@ +.. _development-vagrant: + +======= +Vagrant +======= + +Kayobe provides a Vagrantfile that can be used to bring up a virtual machine +for use as a development environment. The VM is based on the `stackhpc/centos-7 +`_ CentOS 7 image, and +supports the following providers: + +* VirtualBox +* VMWare Fusion + +The VM is configured with 4GB RAM. It has a single private network in addition +to the standard Vagrant NAT network. + +Preparation +=========== + +First, ensure that Vagrant is installed and correctly configured to use +virtual box. Also install the following vagrant plugins:: + + vagrant plugin install vagrant-vbguest + vagrant plugin install vagrant-reload + +Note: if using Ubuntu 16.04 LTS, you may be unable to install any plugins. To +work around this install the upstream version from www.virtualbox.org. + +Usage +===== + +Later sections in the development guide cover in more detail how to use the +development VM in different configurations. These steps cover bringing up and +accessing the VM. + +Clone the kayobe repository:: + + git clone https://github.com/stackhpc/kayobe + +Change the current directory to the kayobe repository:: + + cd kayobe + +Inspect kayobe's ``Vagrantfile``, noting the provisioning steps:: + + less Vagrantfile + +Bring up a virtual machine:: + + vagrant up + +Wait for the VM to boot, then SSH in:: + + vagrant ssh diff --git a/doc/source/index.rst b/doc/source/index.rst index aac175ceb..9e90e5b06 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -42,8 +42,7 @@ Developer Documentation .. toctree:: :maxdepth: 2 - contributing - development + development/index Release Notes -------------