Better init script

Written in Python. Easier to maintain. Easier to make interactive.

Change-Id: Ib579b43c1564b55165de5c2f3d20387122448b19
This commit is contained in:
Pete Vander Giessen 2019-08-07 18:25:18 +00:00
parent b5835060ca
commit 93f412fc93
19 changed files with 978 additions and 321 deletions

1
.gitignore vendored
View File

@ -55,6 +55,7 @@ nosetests.xml
coverage.xml
*.cover
.hypothesis/
.stestr/
# Translations
*.mo

3
.stestr.conf Normal file
View File

@ -0,0 +1,3 @@
[DEFAULT]
test_path=./tools/init/tests/
top_dir=./tools/init/

View File

@ -3,6 +3,8 @@
parent: openstack-tox-snap-with-sudo
timeout: 7200
nodeset: ubuntu-bionic
vars:
tox_envlist: init_lint init_unit snap
- project:
check:

View File

@ -1,317 +0,0 @@
#!/bin/bash
set -e
echo "Initializing Microstack."
##############################################################################
#
# Config
#
# Setup env and templates.
#
##############################################################################
echo "Loading config and writing out templates ..."
ospassword=$(snapctl get ospassword)
extgateway=$(snapctl get extgateway)
extcidr=$(snapctl get extcidr)
dns=$(snapctl get dns)
# Check Config
if [ -z "$ospassword" -o -z "$extgateway" -o -z "$dns" -o -z "$extcidr"]; then
echo "Missing required config value."
exit 1
fi
# Write out templates and read off of our microstack.rc template
# TODO: any password change hooks would go here, updating the password
# in the db before writing it to the templates and restarting
# services.
snap-openstack setup # Write out templates
# Load openstack .rc into this script's environment. Outside of the
# snap shell, this is handled by a wrapper.
source $SNAP_COMMON/etc/microstack.rc
##############################################################################
#
# System Optimization
#
# Perform some tasks that change the host system in ways to better
# support microstack.
#
##############################################################################
# Open up networking so that instances can route to the Internet (see
# bin/setup-br-ex for more networking setup, executed on microstack
# services start.)
echo "Setting up ipv4 forwarding."
sudo sysctl net.ipv4.ip_forward=1
# TODO: add vm swappiness and increased file handle limits here.
# TODO: make vm swappiness and file handle changes optional.
##############################################################################
#
# RabbitMQ Setup
#
# Configure database and wait for services to start.
#
##############################################################################
echo "Configuring RabbitMQ"
echo "Waiting for rabbitmq to start"
while ! nc -z $extgateway 5672; do sleep 0.1; done;
while :;
do
grep "Starting broker..." ${SNAP_COMMON}/log/rabbitmq/startup_log && \
grep "completed" ${SNAP_COMMON}/log/rabbitmq/startup_log && \
break
sleep 1;
done
echo "Rabbitmq started."
# Config!
HOME=$SNAP_COMMON/lib/rabbitmq rabbitmqctl add_user openstack rabbitmq || :
HOME=$SNAP_COMMON/lib/rabbitmq rabbitmqctl set_permissions openstack ".*" ".*" ".*"
##############################################################################
#
# Database setup
#
# Create databases and initialize keystone.
#
##############################################################################
# Wait for MySQL to startup
echo "Waiting for MySQL server to start ..."
while ! nc -z $extgateway 3306; do sleep 0.1; done;
while :;
do
grep "mysqld: ready for connections." \
${SNAP_COMMON}/log/mysql/error.log && break;
sleep 1;
done
echo "Mysql server started."
for db in neutron nova nova_api nova_cell0 cinder glance keystone; do
echo "CREATE DATABASE IF NOT EXISTS ${db}; GRANT ALL PRIVILEGES ON ${db}.* TO '${db}'@'$extgateway' IDENTIFIED BY '${db}';" \
| mysql-start-client -u root
done
# Configure Keystone Fernet Keys
echo "Configuring Keystone..."
snap-openstack launch keystone-manage fernet_setup \
--keystone-user root \
--keystone-group root
snap-openstack launch keystone-manage db_sync
systemctl restart snap.microstack.keystone-*
openstack user show admin || {
snap-openstack launch keystone-manage bootstrap \
--bootstrap-password $ospassword \
--bootstrap-admin-url http://$extgateway:5000/v3/ \
--bootstrap-internal-url http://$extgateway:5000/v3/ \
--bootstrap-public-url http://$extgateway:5000/v3/ \
--bootstrap-region-id microstack
}
openstack project show service || {
openstack project create --domain default --description "Service Project" service
}
echo "Keystone configured."
##############################################################################
#
# Nova Setup
#
# Configure database and wait for services to start.
#
##############################################################################
echo "Configuring Nova..."
openstack user show nova || {
openstack user create --domain default --password nova nova
openstack role add --project service --user nova admin
}
openstack user show placement || {
openstack user create --domain default --password placement placement
openstack role add --project service --user placement admin
}
openstack service show compute || {
openstack service create --name nova \
--description "OpenStack Compute" compute
for endpoint in public internal admin; do
openstack endpoint create --region microstack \
compute $endpoint http://$extgateway:8774/v2.1 || :
done
}
openstack service show placement || {
openstack service create --name placement \
--description "Placement API" placement
for endpoint in public internal admin; do
openstack endpoint create --region microstack \
placement $endpoint http://$extgateway:8778 || :
done
}
# Grant nova user access to cell0
echo "GRANT ALL PRIVILEGES ON nova_cell0.* TO 'nova'@'$extgateway' IDENTIFIED BY 'nova';" \
| mysql-start-client -u root
snap-openstack launch nova-manage api_db sync
snap-openstack launch nova-manage cell_v2 list_cells | grep cell0 || {
snap-openstack launch nova-manage cell_v2 map_cell0
}
snap-openstack launch nova-manage cell_v2 list_cells | grep cell1 || {
snap-openstack launch nova-manage cell_v2 create_cell --name=cell1 --verbose
}
snap-openstack launch nova-manage db sync
systemctl restart snap.microstack.nova-*
while ! nc -z $extgateway 8774; do sleep 0.1; done;
sleep 5
openstack flavor show m1.tiny || {
openstack flavor create --id 1 --ram 512 --disk 1 --vcpus 1 m1.tiny
}
openstack flavor show m1.small || {
openstack flavor create --id 2 --ram 2048 --disk 20 --vcpus 1 m1.small
}
openstack flavor show m1.medium || {
openstack flavor create --id 3 --ram 4096 --disk 20 --vcpus 2 m1.medium
}
openstack flavor show m1.large || {
openstack flavor create --id 4 --ram 8192 --disk 20 --vcpus 4 m1.large
}
openstack flavor show m1.xlarge || {
openstack flavor create --id 5 --ram 16384 --disk 20 --vcpus 8 m1.xlarge
}
##############################################################################
#
# Neutron Setup
#
# Configure database and wait for services to start.
#
##############################################################################
echo "Configuring Neutron"
openstack user show neutron || {
openstack user create --domain default --password neutron neutron
openstack role add --project service --user neutron admin
}
openstack service show network || {
openstack service create --name neutron \
--description "OpenStack Network" network
for endpoint in public internal admin; do
openstack endpoint create --region microstack \
network $endpoint http://$extgateway:9696 || :
done
}
snap-openstack launch neutron-db-manage upgrade head
systemctl restart snap.microstack.neutron-*
while ! nc -z $extgateway 9696; do sleep 0.1; done;
sleep 5
openstack network show test || {
openstack network create test
}
openstack subnet show test-subnet || {
openstack subnet create --network test --subnet-range 192.168.222.0/24 test-subnet
}
openstack network show external || {
openstack network create --external \
--provider-physical-network=physnet1 \
--provider-network-type=flat external
}
openstack subnet show external-subnet || {
openstack subnet create --network external --subnet-range 10.20.20.0/24 \
--no-dhcp external-subnet
}
openstack router show test-router || {
openstack router create test-router
openstack router add subnet test-router test-subnet
openstack router set --external-gateway external test-router
}
##############################################################################
#
# Glance Setup
#
# Configure database and wait for services to start.
#
##############################################################################
echo "Configuring Glance"
openstack user show glance || {
openstack user create --domain default --password glance glance
openstack role add --project service --user glance admin
}
openstack service show image || {
openstack service create --name glance --description "OpenStack Image" image
for endpoint in internal admin public; do
openstack endpoint create --region microstack \
image $endpoint http://$extgateway:9292 || :
done
}
snap-openstack launch glance-manage db_sync
systemctl restart snap.microstack.glance*
while ! nc -z $extgateway 9292; do sleep 0.1; done;
sleep 5
# Setup the cirros image, which is used by the launch app
echo "Grabbing cirros image."
openstack image show cirros || {
[ -f $SNAP_COMMON/images/cirros-0.4.0-x86_64-disk.img ] || {
mkdir -p $SNAP_COMMON/images
wget \
http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img \
-O ${SNAP_COMMON}/images/cirros-0.4.0-x86_64-disk.img
}
openstack image create \
--file ${SNAP_COMMON}/images/cirros-0.4.0-x86_64-disk.img \
--public --container-format=bare --disk-format=qcow2 cirros
}
##############################################################################
#
# Post-setup tasks.
#
# Clean up hanging threads and wait for services to restart.
#
##############################################################################
# Restart libvirt and virtlogd to get logging
# TODO: figure out why this doesn't Just Work initially
systemctl restart snap.microstack.*virt*
echo "Complete. Marking microstack as initialized!"
snapctl set initialized=true

View File

@ -18,6 +18,6 @@ ovs-vsctl --retry --may-exist add-br br-ex
ip address add $extcidr dev br-ex || :
ip link set br-ex up || :
sudo iptables -t nat -A POSTROUTING -s $extcidr ! -d $extcidr -j MASQUERADE
sudo iptables -w -t nat -A POSTROUTING -s $extcidr ! -d $extcidr -j MASQUERADE
exit 0

View File

@ -22,7 +22,7 @@ apps:
# OpenStack Service Configuration
init:
command: init.sh
command: microstack_init
# plugs:
# - network
@ -792,3 +792,12 @@ parts:
overlay:
plugin: dump
source: snap-overlay
# Optionally interactive init script
init:
plugin: python
python-version: python3
python-packages:
- pymysql
- wget
source: tools/init

View File

@ -79,7 +79,7 @@ if [ "${UPGRADE_FROM}" != "none" ]; then
$PREFIX sudo snap install --classic --${UPGRADE_FROM} microstack
fi
# Install the snap under test
# Install the snap under test -- try again if the machine is not yet ready.
$PREFIX sudo snap install --classic --dangerous microstack*.snap
$PREFIX sudo /snap/bin/microstack.init

View File

@ -0,0 +1,13 @@
# Copyright 2019 Canonical Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

52
tools/init/init/config.py Normal file
View File

@ -0,0 +1,52 @@
"""config.py
Keep track of shared config, logging, etc. here.
----------------------------------------------------------------------
Copyright 2019 Canonical Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import logging
import os
import sys
class Env():
"""Singleton that tracks environment variables.
Contains the env variables of the shell that called us. We also
add the snapctl config values to it in the Setup Question.
"""
_global_config = {}
_global_config.update(**os.environ)
def __init__(self):
self.__dict__ = self._global_config
def get_env(self):
"""Get a mapping friendly dict."""
return self.__dict__
logging.basicConfig(
# filename='{SNAP_COMMON}/log/microstack_init.log'.format(**Env),
stream=sys.stdout,
level=logging.DEBUG
)
log = logging # noqa

62
tools/init/init/main.py Normal file
View File

@ -0,0 +1,62 @@
"""Microstack Init
Initialize the databases and configuration files of a microstack
install.
We structure our init in the form of 'Question' classes, each of which
has an 'ask' routine, run in the order laid out in the
question_classes in the main function in this file.
.ask will either ask the user a question, and run the appropriate
routine in the Question class, or simply automatically run a routine
without input from the user (in the case of 'required' questions).
----------------------------------------------------------------------
Copyright 2019 Canonical Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
from init.config import log
from init import questions
def main() -> None:
question_list = [
questions.Setup(),
questions.IpForwarding(),
# The following are not yet implemented:
# questions.VmSwappiness(),
# questions.FileHandleLimits(),
questions.RabbitMQ(),
questions.DatabaseSetup(),
questions.NovaSetup(),
questions.NeutronSetup(),
questions.GlanceSetup(),
questions.PostSetup(),
]
for question in question_list:
try:
question.ask()
except questions.ConfigError as e:
log.critical(e)
sys.exit(1)
if __name__ == '__main__':
main()

129
tools/init/init/question.py Normal file
View File

@ -0,0 +1,129 @@
"""question.py
Contains our Question class, which knows how to ask a question, then
run abitrary code.
----------------------------------------------------------------------
Copyright 2019 Canonical Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from typing import Tuple
class InvalidQuestion(Exception):
"""Exception to raies in the case where a Question subclass has not
been properly implemented.
"""
class InvalidAnswer(Exception):
"""Exception to raise in the case where the user has specified an
invalid answer.
"""
class AnswerNotImplemented(Exception):
"""Exception to raise in the case where a 'yes' or 'no' routine has
not been overriden in the subclass, as required.
"""
class Question():
"""
Ask the user a question, and then run code as appropriate.
Contains a support for always defaulting to yes.
TODO: Add support for finding answers in a config.yaml.
"""
_valid_types = [
'binary', # Yes or No, and variants thereof
'string', # Accept (and sanitize) any string
'auto' # Don't actually ask a question -- just execute self.yes(True)
]
_question = '(required)'
_type = 'auto' # can be binary, string or auto
_invalid_prompt = 'Please answer Yes or No.'
_retries = 3
_input_func = input
def __init__(self):
if self._type not in ['binary', 'string', 'auto']:
raise InvalidQuestion(
'Invalid type {} specified'.format(self._type))
def _validate(self, answer: bytes) -> Tuple[str, bool]:
"""Validate an answer.
:param anwser: raw input from the user.
Returns the answer, and whether or not the answer was valid.
"""
if self._type == 'auto':
return True, True
answer = answer.decode('utf8')
if self._type == 'string':
# TODO Santize this!
return answer, True
# self._type is binary
if answer.lower() in ['y', 'yes']:
return True, True
if answer.lower() in ['n', 'no']:
return False, True
return answer, False
def yes(self, answer: str) -> None:
"""Routine to run if the user answers 'yes' or with a value."""
raise AnswerNotImplemented('You must override this method.')
def no(self, answer: str) -> None:
"""Routine to run if the user answers 'no'"""
raise AnswerNotImplemented('You must override this method.')
def ask(self) -> None:
"""
Ask the user a question.
Run self.yes or self.no as appropriate. Raise an error if the
user cannot specify a valid answer after self._retries tries.
"""
prompt = self._question
for i in range(0, self._retries):
awr, valid = self._validate(
self._type == 'auto' or self._input_func(prompt))
if valid:
if awr:
return self.yes(awr)
return self.no(awr)
prompt = '{} is not valid. {}'.format(awr, self._invalid_prompt)
raise InvalidAnswer('Too many invalid answers.')

View File

@ -0,0 +1,410 @@
"""questions.py
All of our subclasses of Question live here.
We might break this file up into multiple pieces at some point, but
for now, we're keeping things simple (if a big lengthy)
----------------------------------------------------------------------
Copyright 2019 Canonical Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from time import sleep
from os import path
from init.shell import (check, call, check_output, shell, sql, nc_wait,
log_wait, restart, download)
from init.config import Env, log
from init.question import Question
_env = Env().get_env()
class ConfigError(Exception):
"""Suitable error to raise in case there is an issue with the snapctl
config or environment vars.
"""
class Setup(Question):
"""Prepare our environment.
Check to make sure that everything is in place, and populate our
config object and os.environ with the correct values.
"""
def yes(self, answer: str) -> None:
"""Since this is an auto question, we always execute yes."""
log.info('Loading config and writing templates ...')
log.info('Validating config ...')
for key in ['ospassword', 'extgateway', 'extcidr', 'dns']:
val = check_output('snapctl', 'get', key)
if not val:
raise ConfigError(
'Expected config value {} is not set.'.format(key))
_env[key] = val
log.info('Writing out templates ...')
check('snap-openstack', 'setup')
# Parse microstack.rc, and load into _env
# TODO: write something more robust (this breaks on comments
# at end of line.)
mstackrc = '{SNAP_COMMON}/etc/microstack.rc'.format(**_env)
with open(mstackrc, 'r') as rc_file:
for line in rc_file.readlines():
if not line.startswith('export'):
continue
key, val = line[7:].split('=')
_env[key.strip()] = val.strip()
class IpForwarding(Question):
"""Possibly setup IP forwarding."""
_type = 'auto' # Auto for now, to maintain old behavior.
_question = 'Do you wish to setup ip forwarding? (recommended)'
def yes(self, answer: str) -> None:
"""Use sysctl to setup ip forwarding."""
log.info('Setting up ipv4 forwarding...')
check('sysctl', 'net.ipv4.ip_forward=1')
class VmSwappiness(Question):
_type = 'binary'
_question = 'Do you wish to set vm swappiness to 1? (recommended)'
def yes(self, answer: str) -> None:
# TODO
pass
class FileHandleLimits(Question):
_type = 'binary'
_question = 'Do you wish to increase file handle limits? (recommended)'
def yes(self, answer: str) -> None:
# TODO
pass
class RabbitMQ(Question):
"""Wait for Rabbit to start, then setup permissions."""
def _wait(self) -> None:
nc_wait(_env['extgateway'], '5672')
log_file = '{SNAP_COMMON}/log/rabbitmq/startup_log'.format(**_env)
log_wait(log_file, 'completed')
def _configure(self) -> None:
"""Configure RabbitMQ
(actions may have already been run, in which case we fail silently).
"""
# Add Erlang HOME to env.
env = dict(**_env)
env['HOME'] = '{SNAP_COMMON}/lib/rabbitmq'.format(**_env)
# Configure RabbitMQ
call('rabbitmqctl', 'add_user', 'openstack', 'rabbitmq', env=env)
shell('rabbitmqctl set_permissions openstack ".*" ".*" ".*"', env=env)
def yes(self, answer: str) -> None:
log.info('Waiting for RabbitMQ to start ...')
self._wait()
log.info('RabbitMQ started!')
log.info('Configuring RabbitMQ ...')
self._configure()
log.info('RabbitMQ Configured!')
class DatabaseSetup(Question):
"""Setup keystone permissions, then setup all databases."""
def _wait(self) -> None:
nc_wait(_env['extgateway'], '3306')
log_wait('{SNAP_COMMON}/log/mysql/error.log'.format(**_env),
'mysqld: ready for connections.')
def _create_dbs(self) -> None:
for db in ('neutron', 'nova', 'nova_api', 'nova_cell0', 'cinder',
'glance', 'keystone'):
sql("CREATE DATABASE IF NOT EXISTS {db};".format(db=db))
sql(
"GRANT ALL PRIVILEGES ON {db}.* TO {db}@{extgateway} \
IDENTIFIED BY '{db}';".format(db=db, **_env))
def _bootstrap(self) -> None:
if call('openstack', 'user', 'show', 'admin'):
return
bootstrap_url = 'http://{extgateway}:5000/v3/'.format(**_env)
check('snap-openstack', 'launch', 'keystone-manage', 'bootstrap',
'--bootstrap-password', _env['ospassword'],
'--bootstrap-admin-url', bootstrap_url,
'--bootstrap-internal-url', bootstrap_url,
'--bootstrap-public-url', bootstrap_url)
def yes(self, answer: str) -> None:
"""Setup Databases.
Create all the MySQL databases we require, then setup the
fernet keys and create the service project.
"""
log.info('Waiting for MySQL server to start ...')
self._wait()
log.info('Mysql server started! Creating databases ...')
self._create_dbs()
log.info('Configuring Keystone Fernet Keys ...')
check('snap-openstack', 'launch', 'keystone-manage',
'fernet_setup', '--keystone-user', 'root',
'--keystone-group', 'root')
check('snap-openstack', 'launch', 'keystone-manage', 'db_sync')
restart('keystone-*')
log.info('Bootstrapping Keystone ...')
self._bootstrap()
log.info('Creating service project ...')
if not call('openstack', 'project', 'show', 'service'):
check('openstack', 'project', 'create', '--domain',
'default', '--description', 'Service Project',
'service')
log.info('Keystone configured!')
class NovaSetup(Question):
"""Create all relevant nova users and services."""
def _flavors(self) -> None:
"""Create default flavors."""
if not call('openstack', 'flavor', 'show', 'm1.tiny'):
check('openstack', 'flavor', 'create', '--id', '1',
'--ram', '512', '--disk', '1', '--vcpus', '1', 'm1.tiny')
if not call('openstack', 'flavor', 'show', 'm1.small'):
check('openstack', 'flavor', 'create', '--id', '2',
'--ram', '2048', '--disk', '20', '--vcpus', '1', 'm1.small')
if not call('openstack', 'flavor', 'show', 'm1.medium'):
check('openstack', 'flavor', 'create', '--id', '3',
'--ram', '4096', '--disk', '20', '--vcpus', '2', 'm1.medium')
if not call('openstack', 'flavor', 'show', 'm1.large'):
check('openstack', 'flavor', 'create', '--id', '4',
'--ram', '8192', '--disk', '20', '--vcpus', '4', 'm1.large')
if not call('openstack', 'flavor', 'show', 'm1.xlarge'):
check('openstack', 'flavor', 'create', '--id', '5',
'--ram', '16384', '--disk', '20', '--vcpus', '8',
'm1.xlarge')
def yes(self, answer: str) -> None:
log.info('Configuring nova ...')
if not call('openstack', 'user', 'show', 'nova'):
check('openstack', 'user', 'create', '--domain',
'default', '--password', 'nova', 'nova')
check('openstack', 'role', 'add', '--project',
'service', '--user', 'nova', 'admin')
if not call('openstack', 'user', 'show', 'placement'):
check('openstack', 'user', 'create', '--domain', 'default',
'--password', 'placement', 'placement')
check('openstack', 'role', 'add', '--project', 'service',
'--user', 'placement', 'admin')
if not call('openstack', 'service', 'show', 'compute'):
check('openstack', 'service', 'create', '--name', 'nova',
'--description', '"Openstack Compute"', 'compute')
for endpoint in ['public', 'internal', 'admin']:
call('openstack', 'endpoint', 'create', '--region',
'microstack', 'compute', endpoint,
'http://{extgateway}:8774/v2.1'.format(**_env))
if not call('openstack', 'service', 'show', 'placement'):
check('openstack', 'service', 'create', '--name',
'placement', '--description', '"Placement API"',
'placement')
for endpoint in ['public', 'internal', 'admin']:
call('openstack', 'endpoint', 'create', '--region',
'microstack', 'placement', endpoint,
'http://{extgateway}:8778'.format(**_env))
# Grant nova user access to cell0
sql(
"GRANT ALL PRIVILEGES ON nova_cell0.* TO 'nova'@'{extgateway}' \
IDENTIFIED BY \'nova';".format(**_env))
check('snap-openstack', 'launch', 'nova-manage', 'api_db', 'sync')
if 'cell0' not in check_output('snap-openstack', 'launch',
'nova-manage', 'cell_v2',
'list_cells'):
check('snap-openstack', 'launch', 'nova-manage',
'cell_v2', 'map_cell0')
if 'cell1' not in check_output('snap-openstack', 'launch',
'nova-manage', 'cell_v2', 'list_cells'):
check('snap-openstack', 'launch', 'nova-manage', 'cell_v2',
'create_cell', '--name=cell1', '--verbose')
check('snap-openstack', 'launch', 'nova-manage', 'db', 'sync')
restart('nova-*')
nc_wait(_env['extgateway'], '8774')
sleep(5) # TODO: log_wait
log.info('Creating default flavors...')
self._flavors()
class NeutronSetup(Question):
"""Create all relevant neutron services and users."""
def yes(self, answer: str) -> None:
log.info('Configuring Neutron')
if not call('openstack', 'user', 'show', 'neutron'):
check('openstack', 'user', 'create', '--domain', 'default',
'--password', 'neutron', 'neutron')
check('openstack', 'role', 'add', '--project', 'service',
'--user', 'neutron', 'admin')
if not call('openstack', 'service', 'show', 'network'):
check('openstack', 'service', 'create', '--name', 'neutron',
'--description', '"OpenStack Network"', 'network')
for endpoint in ['public', 'internal', 'admin']:
call('openstack', 'endpoint', 'create', '--region',
'microstack', 'network', endpoint,
'http://{extgateway}:9696'.format(**_env))
check('snap-openstack', 'launch', 'neutron-db-manage', 'upgrade',
'head')
restart('neutron-*')
nc_wait(_env['extgateway'], '9696')
sleep(5) # TODO: log_wait
if not call('openstack', 'network', 'show', 'test'):
check('openstack', 'network', 'create', 'test')
if not call('openstack', 'subnet', 'show', 'test-subnet'):
check('openstack', 'subnet', 'create', '--network', 'test',
'--subnet-range', '192.168.222.0/24', 'test-subnet')
if not call('openstack', 'network', 'show', 'external'):
check('openstack', 'network', 'create', '--external',
'--provider-physical-network=physnet1',
'--provider-network-type=flat', 'external')
if not call('openstack', 'subnet', 'show', 'external-subnet'):
check('openstack', 'subnet', 'create', '--network', 'external',
'--subnet-range', _env['extcidr'], '--no-dhcp',
'external-subnet')
if not call('openstack', 'router', 'show', 'test-router'):
check('openstack', 'router', 'create', 'test-router')
check('openstack', 'router', 'add', 'subnet', 'test-router',
'test-subnet')
check('openstack', 'router', 'set', '--external-gateway',
'external', 'test-router')
class GlanceSetup(Question):
"""Setup glance, and download an initial Cirros image."""
def _fetch_cirros(self) -> None:
if call('openstack', 'image', 'show', 'cirros'):
return
env = dict(**_env)
env['VER'] = '0.4.0'
env['IMG'] = 'cirros-{VER}-x86_64-disk.img'.format(**env)
log.info('Fetching cirros image ...')
cirros_path = '{SNAP_COMMON}/images/{IMG}'.format(**env)
if not path.exists(cirros_path):
check('mkdir', '-p', '{SNAP_COMMON}/images'.format(**env))
download(
'http://download.cirros-cloud.net/{VER}/{IMG}'.format(**env),
'{SNAP_COMMON}/images/{IMG}'.format(**env))
check('openstack', 'image', 'create', '--file',
'{SNAP_COMMON}/images/{IMG}'.format(**env),
'--public', '--container-format=bare',
'--disk-format=qcow2', 'cirros')
def yes(self, answer: str) -> None:
log.info('Configuring Glance ...')
if not call('openstack', 'user', 'show', 'glance'):
check('openstack', 'user', 'create', '--domain', 'default',
'--password', 'glance', 'glance')
check('openstack', 'role', 'add', '--project', 'service',
'--user', 'glance', 'admin')
if not call('openstack', 'service', 'show', 'image'):
check('openstack', 'service', 'create', '--name', 'glance',
'--description', '"OpenStack Image"', 'image')
for endpoint in ['internal', 'admin', 'public']:
check('openstack', 'endpoint', 'create', '--region',
'microstack', 'image', endpoint,
'http://{extgateway}:9292'.format(**_env))
check('snap-openstack', 'launch', 'glance-manage', 'db_sync')
restart('glance*')
nc_wait(_env['extgateway'], '9292')
sleep(5) # TODO: log_wait
self._fetch_cirros()
class PostSetup(Question):
"""Sneak in any additional cleanup, then set the initialized state."""
def yes(self, answer: str) -> None:
log.info('restarting libvirt and virtlogd ...')
# This fixes an issue w/ logging not getting set.
# TODO: fix issue.
restart('*virt*')
check('snapctl', 'set', 'initialized=true')
log.info('Complete. Marked microstack as initialized!')

136
tools/init/init/shell.py Normal file
View File

@ -0,0 +1,136 @@
"""shell.py
Contains wrappers around subprocess and pymysql commands, specific to
our needs in the init script.
# TODO capture stdout (and output to log.DEBUG)
----------------------------------------------------------------------
Copyright 2019 Canonical Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import subprocess
from time import sleep
from typing import Dict, List
import pymysql
import wget
from init.config import Env
_env = Env().get_env()
def check(*args: List[str], env: Dict = _env) -> int:
"""Execute a shell command, raising an error on failed excution.
:param args: strings to be composed into the bash call.
:param env: defaults to our Env singleton; can be overriden.
"""
return subprocess.check_call(args, env=env)
def check_output(*args: List[str], env: Dict = _env) -> str:
"""Execute a shell command, returning the output of the command.
:param args: strings to be composed into the bash call.
:param env: defaults to our Env singleton; can be overriden.
Include our env; pass in any extra keyword args.
"""
return subprocess.check_output(args, env=env,
universal_newlines=True).strip()
def call(*args: List[str], env: Dict = _env) -> bool:
"""Execute a shell command.
Return True if the call executed successfully (returned 0), or
False if it returned with an error code (return > 0)
:param args: strings to be composed into the bash call.
:param env: defaults to our Env singleton; can be overriden.
"""
return not subprocess.call(args, env=env)
def shell(cmd: str, env: Dict = _env) -> int:
"""Execute a command, using the actual bourne again shell.
Use this in cases where it is difficult to compose a comma
separate list that will get parsed into a succesful bash
command. (E.g., your bash command contains an argument like ".*"
".*" ".*")
:param cmd: the command to run.
:param env: defaults to our Env singleton; can be overriden.
"""
return subprocess.check_call(cmd, shell=True, env=env,
executable='/snap/core18/current/bin/bash')
def sql(cmd: str) -> None:
"""Execute some SQL!
Really simply wrapper around a pymysql connection, suitable for
passing the limited CREATE and GRANT commands that we need to pass
in our init script.
:param cmd: sql to execute.
"""
mysql_conf = '${SNAP_USER_COMMON}/etc/mysql/my.cnf'.format(**_env)
connection = pymysql.connect(host='localhost', user='root',
read_default_file=mysql_conf)
with connection.cursor() as cursor:
cursor.execute(cmd)
def nc_wait(addr: str, port: str) -> None:
"""Wait for a service to be answering on a port."""
print('Waiting for {}:{}'.format(addr, port))
while not call('nc', '-z', addr, port):
sleep(1)
def log_wait(log: str, message: str) -> None:
"""Wait until a message appears in a log."""
while True:
with open(log, 'r') as log_file:
for line in log_file.readlines():
if message in line:
return
sleep(1)
def restart(service: str) -> None:
"""Restart a microstack service.
:param service: the service(s) to be restarted. Can contain wild cards.
e.g. *rabbit*
"""
check('systemctl', 'restart', 'snap.microstack.{}'.format(service))
def download(url: str, output: str) -> None:
"""Download a file to a path"""
wget.download(url, output)

View File

@ -0,0 +1,2 @@
pymysql
wget

13
tools/init/setup.py Normal file
View File

@ -0,0 +1,13 @@
from setuptools import setup, find_packages
setup(
name="microstack_init",
description="Optionally interactive init script for Microstack.",
packages=find_packages(),
version="0.0.1",
entry_points={
'console_scripts': [
'microstack_init = init.main:main',
],
},
)

View File

@ -0,0 +1,4 @@
pylint
pep8
stestr
flake8

View File

View File

@ -0,0 +1,125 @@
import sys
import os
import unittest
# TODO: drop in test runner and get rid of this line.
sys.path.append(os.getcwd()) # noqa
from init.question import (Question, InvalidQuestion, InvalidAnswer,
AnswerNotImplemented)
##############################################################################
#
# Test Fixtures
#
##############################################################################
class InvalidTypeQuestion(Question):
_type = 'foo'
class IncompleteQuestion(Question):
_type = 'auto'
class GoodAutoQuestion(Question):
_type = 'auto'
def yes(self, answer):
return 'I am a good question!'
class GoodBinaryQuestion(Question):
_type = 'binary'
def yes(self, answer):
return True
def no(self, answer):
return False
class GoodStringQuestion(Question):
"""Pass a string through to the output of Question.ask.
# TODO right now, we have separate handlers for Truthy and Falsey
answers, and this test class basically makes them do the same
thing. Is this a good pattern?
"""
_type = 'string'
def yes(self, answer):
return answer
def no(self, answer):
return answer
##############################################################################
#
# Tests Proper
#
##############################################################################
class TestQuestionClass(unittest.TestCase):
"""
Test basic features of the Question class.
"""
def test_invalid_type(self):
with self.assertRaises(InvalidQuestion):
InvalidTypeQuestion().ask()
def test_valid_type(self):
self.assertTrue(GoodBinaryQuestion())
def test_not_implemented(self):
with self.assertRaises(AnswerNotImplemented):
IncompleteQuestion().ask()
def test_auto_question(self):
self.assertEqual(GoodAutoQuestion().ask(), 'I am a good question!')
class TestInput(unittest.TestCase):
"""
Test input handling.
Takes advantage of the fact that we can override the Question
class's input handler.
"""
def test_binary_question(self):
q = GoodBinaryQuestion()
for answer in ['yes', 'Yes', 'y']:
q._input_func = lambda x: answer.encode('utf8')
self.assertTrue(q.ask())
for answer in ['No', 'n', 'no']:
q._input_func = lambda x: answer.encode('utf8')
self.assertFalse(q.ask())
with self.assertRaises(InvalidAnswer):
q._input_func = lambda x: 'foo'.encode('utf8')
q.ask()
def test_string_question(self):
q = GoodStringQuestion()
for answer in ['foo', 'bar', 'baz', '', 'yadayadayada']:
q._input_func = lambda x: answer.encode('utf8')
self.assertEqual(answer, q.ask())
if __name__ == '__main__':
unittest.main()

15
tox.ini
View File

@ -1,5 +1,5 @@
[tox]
envlist = multipass
envlist = init_lint, init_unit, multipass
skipsdist = True
[testenv]
@ -36,3 +36,16 @@ commands =
# Just run basic_test.sh, with multipass support.
commands =
{toxinidir}/tests/basic-test.sh -m
[testenv:init_lint]
basepython=python3
deps = -r{toxinidir}/tools/init/test-requirements.txt
-r{toxinidir}/tools/init/requirements.txt
commands = flake8 {toxinidir}/tools/init/init/
[testenv:init_unit]
basepython=python3
deps = -r{toxinidir}/tools/init/test-requirements.txt
-r{toxinidir}/tools/init/requirements.txt
commands = stestr run {posargs}