Written in Python. Easier to maintain. Easier to make interactive. Change-Id: Ib579b43c1564b55165de5c2f3d20387122448b19changes/90/675390/16
Pete Vander Giessen
3 years ago
parent
b5835060ca
commit
93f412fc93
@ -0,0 +1,3 @@ |
||||
[DEFAULT] |
||||
test_path=./tools/init/tests/ |
||||
top_dir=./tools/init/ |
||||
@ -1,317 +0,0 @@ |
||||
#!/bin/bash |
||||
set -e |
||||
|
||||
echo "Initializing Microstack." |
||||
|
||||
############################################################################## |
||||
# |
||||
# Config |
||||
# |
||||
# Setup env and templates. |
||||
# |
||||
############################################################################## |
||||
echo "Loading config and writing out templates ..." |
||||
|
||||
ospassword=$(snapctl get ospassword) |
||||
extgateway=$(snapctl get extgateway) |
||||
extcidr=$(snapctl get extcidr) |
||||
dns=$(snapctl get dns) |
||||
|
||||
# Check Config |
||||
if [ -z "$ospassword" -o -z "$extgateway" -o -z "$dns" -o -z "$extcidr"]; then |
||||
echo "Missing required config value." |
||||
exit 1 |
||||
fi |
||||
|
||||
# Write out templates and read off of our microstack.rc template |
||||
# TODO: any password change hooks would go here, updating the password |
||||
# in the db before writing it to the templates and restarting |
||||
# services. |
||||
snap-openstack setup # Write out templates |
||||
|
||||
# Load openstack .rc into this script's environment. Outside of the |
||||
# snap shell, this is handled by a wrapper. |
||||
source $SNAP_COMMON/etc/microstack.rc |
||||
|
||||
|
||||
############################################################################## |
||||
# |
||||
# System Optimization |
||||
# |
||||
# Perform some tasks that change the host system in ways to better |
||||
# support microstack. |
||||
# |
||||
############################################################################## |
||||
|
||||
|
||||
# Open up networking so that instances can route to the Internet (see |
||||
# bin/setup-br-ex for more networking setup, executed on microstack |
||||
# services start.) |
||||
echo "Setting up ipv4 forwarding." |
||||
sudo sysctl net.ipv4.ip_forward=1 |
||||
|
||||
# TODO: add vm swappiness and increased file handle limits here. |
||||
# TODO: make vm swappiness and file handle changes optional. |
||||
|
||||
|
||||
############################################################################## |
||||
# |
||||
# RabbitMQ Setup |
||||
# |
||||
# Configure database and wait for services to start. |
||||
# |
||||
############################################################################## |
||||
echo "Configuring RabbitMQ" |
||||
|
||||
echo "Waiting for rabbitmq to start" |
||||
while ! nc -z $extgateway 5672; do sleep 0.1; done; |
||||
while :; |
||||
do |
||||
grep "Starting broker..." ${SNAP_COMMON}/log/rabbitmq/startup_log && \ |
||||
grep "completed" ${SNAP_COMMON}/log/rabbitmq/startup_log && \ |
||||
break |
||||
sleep 1; |
||||
done |
||||
echo "Rabbitmq started." |
||||
|
||||
# Config! |
||||
HOME=$SNAP_COMMON/lib/rabbitmq rabbitmqctl add_user openstack rabbitmq || : |
||||
HOME=$SNAP_COMMON/lib/rabbitmq rabbitmqctl set_permissions openstack ".*" ".*" ".*" |
||||
|
||||
|
||||
############################################################################## |
||||
# |
||||
# Database setup |
||||
# |
||||
# Create databases and initialize keystone. |
||||
# |
||||
############################################################################## |
||||
|
||||
# Wait for MySQL to startup |
||||
echo "Waiting for MySQL server to start ..." |
||||
while ! nc -z $extgateway 3306; do sleep 0.1; done; |
||||
while :; |
||||
do |
||||
grep "mysqld: ready for connections." \ |
||||
${SNAP_COMMON}/log/mysql/error.log && break; |
||||
sleep 1; |
||||
done |
||||
echo "Mysql server started." |
||||
|
||||
for db in neutron nova nova_api nova_cell0 cinder glance keystone; do |
||||
echo "CREATE DATABASE IF NOT EXISTS ${db}; GRANT ALL PRIVILEGES ON ${db}.* TO '${db}'@'$extgateway' IDENTIFIED BY '${db}';" \ |
||||
| mysql-start-client -u root |
||||
done |
||||
|
||||
# Configure Keystone Fernet Keys |
||||
echo "Configuring Keystone..." |
||||
snap-openstack launch keystone-manage fernet_setup \ |
||||
--keystone-user root \ |
||||
--keystone-group root |
||||
snap-openstack launch keystone-manage db_sync |
||||
|
||||
systemctl restart snap.microstack.keystone-* |
||||
|
||||
openstack user show admin || { |
||||
snap-openstack launch keystone-manage bootstrap \ |
||||
--bootstrap-password $ospassword \ |
||||
--bootstrap-admin-url http://$extgateway:5000/v3/ \ |
||||
--bootstrap-internal-url http://$extgateway:5000/v3/ \ |
||||
--bootstrap-public-url http://$extgateway:5000/v3/ \ |
||||
--bootstrap-region-id microstack |
||||
} |
||||
|
||||
openstack project show service || { |
||||
openstack project create --domain default --description "Service Project" service |
||||
} |
||||
echo "Keystone configured." |
||||
|
||||
############################################################################## |
||||
# |
||||
# Nova Setup |
||||
# |
||||
# Configure database and wait for services to start. |
||||
# |
||||
############################################################################## |
||||
echo "Configuring Nova..." |
||||
|
||||
openstack user show nova || { |
||||
openstack user create --domain default --password nova nova |
||||
openstack role add --project service --user nova admin |
||||
} |
||||
|
||||
openstack user show placement || { |
||||
openstack user create --domain default --password placement placement |
||||
openstack role add --project service --user placement admin |
||||
} |
||||
|
||||
openstack service show compute || { |
||||
openstack service create --name nova \ |
||||
--description "OpenStack Compute" compute |
||||
|
||||
for endpoint in public internal admin; do |
||||
openstack endpoint create --region microstack \ |
||||
compute $endpoint http://$extgateway:8774/v2.1 || : |
||||
done |
||||
} |
||||
|
||||
openstack service show placement || { |
||||
openstack service create --name placement \ |
||||
--description "Placement API" placement |
||||
|
||||
for endpoint in public internal admin; do |
||||
openstack endpoint create --region microstack \ |
||||
placement $endpoint http://$extgateway:8778 || : |
||||
done |
||||
} |
||||
|
||||
# Grant nova user access to cell0 |
||||
echo "GRANT ALL PRIVILEGES ON nova_cell0.* TO 'nova'@'$extgateway' IDENTIFIED BY 'nova';" \ |
||||
| mysql-start-client -u root |
||||
|
||||
snap-openstack launch nova-manage api_db sync |
||||
snap-openstack launch nova-manage cell_v2 list_cells | grep cell0 || { |
||||
snap-openstack launch nova-manage cell_v2 map_cell0 |
||||
} |
||||
snap-openstack launch nova-manage cell_v2 list_cells | grep cell1 || { |
||||
snap-openstack launch nova-manage cell_v2 create_cell --name=cell1 --verbose |
||||
} |
||||
snap-openstack launch nova-manage db sync |
||||
|
||||
systemctl restart snap.microstack.nova-* |
||||
|
||||
while ! nc -z $extgateway 8774; do sleep 0.1; done; |
||||
|
||||
sleep 5 |
||||
|
||||
openstack flavor show m1.tiny || { |
||||
openstack flavor create --id 1 --ram 512 --disk 1 --vcpus 1 m1.tiny |
||||
} |
||||
openstack flavor show m1.small || { |
||||
openstack flavor create --id 2 --ram 2048 --disk 20 --vcpus 1 m1.small |
||||
} |
||||
openstack flavor show m1.medium || { |
||||
openstack flavor create --id 3 --ram 4096 --disk 20 --vcpus 2 m1.medium |
||||
} |
||||
openstack flavor show m1.large || { |
||||
openstack flavor create --id 4 --ram 8192 --disk 20 --vcpus 4 m1.large |
||||
} |
||||
openstack flavor show m1.xlarge || { |
||||
openstack flavor create --id 5 --ram 16384 --disk 20 --vcpus 8 m1.xlarge |
||||
} |
||||
|
||||
############################################################################## |
||||
# |
||||
# Neutron Setup |
||||
# |
||||
# Configure database and wait for services to start. |
||||
# |
||||
############################################################################## |
||||
echo "Configuring Neutron" |
||||
|
||||
openstack user show neutron || { |
||||
openstack user create --domain default --password neutron neutron |
||||
openstack role add --project service --user neutron admin |
||||
} |
||||
|
||||
openstack service show network || { |
||||
openstack service create --name neutron \ |
||||
--description "OpenStack Network" network |
||||
|
||||
for endpoint in public internal admin; do |
||||
openstack endpoint create --region microstack \ |
||||
network $endpoint http://$extgateway:9696 || : |
||||
done |
||||
} |
||||
|
||||
snap-openstack launch neutron-db-manage upgrade head |
||||
|
||||
systemctl restart snap.microstack.neutron-* |
||||
|
||||
while ! nc -z $extgateway 9696; do sleep 0.1; done; |
||||
|
||||
sleep 5 |
||||
|
||||
openstack network show test || { |
||||
openstack network create test |
||||
} |
||||
|
||||
openstack subnet show test-subnet || { |
||||
openstack subnet create --network test --subnet-range 192.168.222.0/24 test-subnet |
||||
} |
||||
|
||||
openstack network show external || { |
||||
openstack network create --external \ |
||||
--provider-physical-network=physnet1 \ |
||||
--provider-network-type=flat external |
||||
} |
||||
|
||||
openstack subnet show external-subnet || { |
||||
openstack subnet create --network external --subnet-range 10.20.20.0/24 \ |
||||
--no-dhcp external-subnet |
||||
} |
||||
|
||||
openstack router show test-router || { |
||||
openstack router create test-router |
||||
openstack router add subnet test-router test-subnet |
||||
openstack router set --external-gateway external test-router |
||||
} |
||||
|
||||
############################################################################## |
||||
# |
||||
# Glance Setup |
||||
# |
||||
# Configure database and wait for services to start. |
||||
# |
||||
############################################################################## |
||||
echo "Configuring Glance" |
||||
|
||||
openstack user show glance || { |
||||
openstack user create --domain default --password glance glance |
||||
openstack role add --project service --user glance admin |
||||
} |
||||
|
||||
openstack service show image || { |
||||
openstack service create --name glance --description "OpenStack Image" image |
||||
for endpoint in internal admin public; do |
||||
openstack endpoint create --region microstack \ |
||||
image $endpoint http://$extgateway:9292 || : |
||||
done |
||||
} |
||||
|
||||
snap-openstack launch glance-manage db_sync |
||||
|
||||
systemctl restart snap.microstack.glance* |
||||
|
||||
while ! nc -z $extgateway 9292; do sleep 0.1; done; |
||||
|
||||
sleep 5 |
||||
|
||||
# Setup the cirros image, which is used by the launch app |
||||
echo "Grabbing cirros image." |
||||
openstack image show cirros || { |
||||
[ -f $SNAP_COMMON/images/cirros-0.4.0-x86_64-disk.img ] || { |
||||
mkdir -p $SNAP_COMMON/images |
||||
wget \ |
||||
http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img \ |
||||
-O ${SNAP_COMMON}/images/cirros-0.4.0-x86_64-disk.img |
||||
} |
||||
openstack image create \ |
||||
--file ${SNAP_COMMON}/images/cirros-0.4.0-x86_64-disk.img \ |
||||
--public --container-format=bare --disk-format=qcow2 cirros |
||||
} |
||||
|
||||
############################################################################## |
||||
# |
||||
# Post-setup tasks. |
||||
# |
||||
# Clean up hanging threads and wait for services to restart. |
||||
# |
||||
############################################################################## |
||||
|
||||
# Restart libvirt and virtlogd to get logging |
||||
# TODO: figure out why this doesn't Just Work initially |
||||
systemctl restart snap.microstack.*virt* |
||||
|
||||
echo "Complete. Marking microstack as initialized!" |
||||
snapctl set initialized=true |
||||
@ -0,0 +1,13 @@ |
||||
# Copyright 2019 Canonical Ltd |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
@ -0,0 +1,52 @@ |
||||
"""config.py |
||||
|
||||
Keep track of shared config, logging, etc. here. |
||||
|
||||
---------------------------------------------------------------------- |
||||
|
||||
Copyright 2019 Canonical Ltd |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
|
||||
""" |
||||
import logging |
||||
import os |
||||
import sys |
||||
|
||||
|
||||
class Env(): |
||||
"""Singleton that tracks environment variables. |
||||
|
||||
Contains the env variables of the shell that called us. We also |
||||
add the snapctl config values to it in the Setup Question. |
||||
|
||||
""" |
||||
_global_config = {} |
||||
_global_config.update(**os.environ) |
||||
|
||||
def __init__(self): |
||||
self.__dict__ = self._global_config |
||||
|
||||
def get_env(self): |
||||
"""Get a mapping friendly dict.""" |
||||
return self.__dict__ |
||||
|
||||
|
||||
logging.basicConfig( |
||||
# filename='{SNAP_COMMON}/log/microstack_init.log'.format(**Env), |
||||
stream=sys.stdout, |
||||
level=logging.DEBUG |
||||
) |
||||
|
||||
|
||||
log = logging # noqa |
||||
@ -0,0 +1,62 @@ |
||||
"""Microstack Init |
||||
|
||||
Initialize the databases and configuration files of a microstack |
||||
install. |
||||
|
||||
We structure our init in the form of 'Question' classes, each of which |
||||
has an 'ask' routine, run in the order laid out in the |
||||
question_classes in the main function in this file. |
||||
|
||||
.ask will either ask the user a question, and run the appropriate |
||||
routine in the Question class, or simply automatically run a routine |
||||
without input from the user (in the case of 'required' questions). |
||||
|
||||
---------------------------------------------------------------------- |
||||
|
||||
Copyright 2019 Canonical Ltd |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
|
||||
""" |
||||
import sys |
||||
|
||||
from init.config import log |
||||
|
||||
from init import questions |
||||
|
||||
|
||||
def main() -> None: |
||||
question_list = [ |
||||
questions.Setup(), |
||||
questions.IpForwarding(), |
||||
# The following are not yet implemented: |
||||
# questions.VmSwappiness(), |
||||
# questions.FileHandleLimits(), |
||||
questions.RabbitMQ(), |
||||
questions.DatabaseSetup(), |
||||
questions.NovaSetup(), |
||||
questions.NeutronSetup(), |
||||
questions.GlanceSetup(), |
||||
questions.PostSetup(), |
||||
] |
||||
|
||||
for question in question_list: |
||||
try: |
||||
question.ask() |
||||
except questions.ConfigError as e: |
||||
log.critical(e) |
||||
sys.exit(1) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
main() |
||||
@ -0,0 +1,129 @@ |
||||
"""question.py |
||||
|
||||
Contains our Question class, which knows how to ask a question, then |
||||
run abitrary code. |
||||
|
||||
---------------------------------------------------------------------- |
||||
|
||||
Copyright 2019 Canonical Ltd |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
|
||||
""" |
||||
|
||||
|
||||
from typing import Tuple |
||||
|
||||
|
||||
class InvalidQuestion(Exception): |
||||
"""Exception to raies in the case where a Question subclass has not |
||||
been properly implemented. |
||||
|
||||
""" |
||||
|
||||
|
||||
class InvalidAnswer(Exception): |
||||
"""Exception to raise in the case where the user has specified an |
||||
invalid answer. |
||||
|
||||
""" |
||||
|
||||
|
||||
class AnswerNotImplemented(Exception): |
||||
"""Exception to raise in the case where a 'yes' or 'no' routine has |
||||
not been overriden in the subclass, as required. |
||||
|
||||
""" |
||||
|
||||
|
||||
class Question(): |
||||
""" |
||||
Ask the user a question, and then run code as appropriate. |
||||
|
||||
Contains a support for always defaulting to yes. |
||||
|
||||
TODO: Add support for finding answers in a config.yaml. |
||||
|
||||
""" |
||||
_valid_types = [ |
||||
'binary', # Yes or No, and variants thereof |
||||
'string', # Accept (and sanitize) any string |
||||
'auto' # Don't actually ask a question -- just execute self.yes(True) |
||||
] |
||||
|
||||
_question = '(required)' |
||||
_type = 'auto' # can be binary, string or auto |
||||
_invalid_prompt = 'Please answer Yes or No.' |
||||
_retries = 3 |
||||
_input_func = input |
||||
|
||||
def __init__(self): |
||||
|
||||
if self._type not in ['binary', 'string', 'auto']: |
||||
raise InvalidQuestion( |
||||
'Invalid type {} specified'.format(self._type)) |
||||
|
||||
def _validate(self, answer: bytes) -> Tuple[str, bool]: |
||||
"""Validate an answer. |
||||
|
||||
:param anwser: raw input from the user. |
||||
|
||||
Returns the answer, and whether or not the answer was valid. |
||||
|
||||
""" |
||||
if self._type == 'auto': |
||||
return True, True |
||||
|
||||
answer = answer.decode('utf8') |
||||
|
||||
if self._type == 'string': |
||||
# TODO Santize this! |
||||
return answer, True |
||||
|
||||
# self._type is binary |
||||
if answer.lower() in ['y', 'yes']: |
||||
return True, True |
||||
|
||||
if answer.lower() in ['n', 'no']: |
||||
return False, True |
||||
|
||||
return answer, False |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
"""Routine to run if the user answers 'yes' or with a value.""" |
||||
raise AnswerNotImplemented('You must override this method.') |
||||
|
||||
def no(self, answer: str) -> None: |
||||
"""Routine to run if the user answers 'no'""" |
||||
raise AnswerNotImplemented('You must override this method.') |
||||
|
||||
def ask(self) -> None: |
||||
""" |
||||
Ask the user a question. |
||||
|
||||
Run self.yes or self.no as appropriate. Raise an error if the |
||||
user cannot specify a valid answer after self._retries tries. |
||||
|
||||
""" |
||||
prompt = self._question |
||||
|
||||
for i in range(0, self._retries): |
||||
awr, valid = self._validate( |
||||
self._type == 'auto' or self._input_func(prompt)) |
||||
if valid: |
||||
if awr: |
||||
return self.yes(awr) |
||||
return self.no(awr) |
||||
prompt = '{} is not valid. {}'.format(awr, self._invalid_prompt) |
||||
|
||||
raise InvalidAnswer('Too many invalid answers.') |
||||
@ -0,0 +1,410 @@ |
||||
"""questions.py |
||||
|
||||
All of our subclasses of Question live here. |
||||
|
||||
We might break this file up into multiple pieces at some point, but |
||||
for now, we're keeping things simple (if a big lengthy) |
||||
|
||||
---------------------------------------------------------------------- |
||||
|
||||
Copyright 2019 Canonical Ltd |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
|
||||
""" |
||||
|
||||
|
||||
from time import sleep |
||||
from os import path |
||||
|
||||
from init.shell import (check, call, check_output, shell, sql, nc_wait, |
||||
log_wait, restart, download) |
||||
from init.config import Env, log |
||||
from init.question import Question |
||||
|
||||
|
||||
_env = Env().get_env() |
||||
|
||||
|
||||
class ConfigError(Exception): |
||||
"""Suitable error to raise in case there is an issue with the snapctl |
||||
config or environment vars. |
||||
|
||||
""" |
||||
|
||||
|
||||
class Setup(Question): |
||||
"""Prepare our environment. |
||||
|
||||
Check to make sure that everything is in place, and populate our |
||||
config object and os.environ with the correct values. |
||||
|
||||
""" |
||||
def yes(self, answer: str) -> None: |
||||
"""Since this is an auto question, we always execute yes.""" |
||||
log.info('Loading config and writing templates ...') |
||||
|
||||
log.info('Validating config ...') |
||||
for key in ['ospassword', 'extgateway', 'extcidr', 'dns']: |
||||
val = check_output('snapctl', 'get', key) |
||||
if not val: |
||||
raise ConfigError( |
||||
'Expected config value {} is not set.'.format(key)) |
||||
_env[key] = val |
||||
|
||||
log.info('Writing out templates ...') |
||||
check('snap-openstack', 'setup') |
||||
|
||||
# Parse microstack.rc, and load into _env |
||||
# TODO: write something more robust (this breaks on comments |
||||
# at end of line.) |
||||
mstackrc = '{SNAP_COMMON}/etc/microstack.rc'.format(**_env) |
||||
with open(mstackrc, 'r') as rc_file: |
||||
for line in rc_file.readlines(): |
||||
if not line.startswith('export'): |
||||
continue |
||||
key, val = line[7:].split('=') |
||||
_env[key.strip()] = val.strip() |
||||
|
||||
|
||||
class IpForwarding(Question): |
||||
"""Possibly setup IP forwarding.""" |
||||
|
||||
_type = 'auto' # Auto for now, to maintain old behavior. |
||||
_question = 'Do you wish to setup ip forwarding? (recommended)' |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
"""Use sysctl to setup ip forwarding.""" |
||||
log.info('Setting up ipv4 forwarding...') |
||||
|
||||
check('sysctl', 'net.ipv4.ip_forward=1') |
||||
|
||||
|
||||
class VmSwappiness(Question): |
||||
|
||||
_type = 'binary' |
||||
_question = 'Do you wish to set vm swappiness to 1? (recommended)' |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
# TODO |
||||
pass |
||||
|
||||
|
||||
class FileHandleLimits(Question): |
||||
|
||||
_type = 'binary' |
||||
_question = 'Do you wish to increase file handle limits? (recommended)' |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
# TODO |
||||
pass |
||||
|
||||
|
||||
class RabbitMQ(Question): |
||||
"""Wait for Rabbit to start, then setup permissions.""" |
||||
|
||||
def _wait(self) -> None: |
||||
nc_wait(_env['extgateway'], '5672') |
||||
log_file = '{SNAP_COMMON}/log/rabbitmq/startup_log'.format(**_env) |
||||
log_wait(log_file, 'completed') |
||||
|
||||
def _configure(self) -> None: |
||||
"""Configure RabbitMQ |
||||
|
||||
(actions may have already been run, in which case we fail silently). |
||||
""" |
||||
# Add Erlang HOME to env. |
||||
env = dict(**_env) |
||||
env['HOME'] = '{SNAP_COMMON}/lib/rabbitmq'.format(**_env) |
||||
# Configure RabbitMQ |
||||
call('rabbitmqctl', 'add_user', 'openstack', 'rabbitmq', env=env) |
||||
shell('rabbitmqctl set_permissions openstack ".*" ".*" ".*"', env=env) |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
log.info('Waiting for RabbitMQ to start ...') |
||||
self._wait() |
||||
log.info('RabbitMQ started!') |
||||
log.info('Configuring RabbitMQ ...') |
||||
self._configure() |
||||
log.info('RabbitMQ Configured!') |
||||
|
||||
|
||||
class DatabaseSetup(Question): |
||||
"""Setup keystone permissions, then setup all databases.""" |
||||
|
||||
def _wait(self) -> None: |
||||
nc_wait(_env['extgateway'], '3306') |
||||
log_wait('{SNAP_COMMON}/log/mysql/error.log'.format(**_env), |
||||
'mysqld: ready for connections.') |
||||
|
||||
def _create_dbs(self) -> None: |
||||
for db in ('neutron', 'nova', 'nova_api', 'nova_cell0', 'cinder', |
||||
'glance', 'keystone'): |
||||
sql("CREATE DATABASE IF NOT EXISTS {db};".format(db=db)) |
||||
sql( |
||||
"GRANT ALL PRIVILEGES ON {db}.* TO {db}@{extgateway} \ |
||||
IDENTIFIED BY '{db}';".format(db=db, **_env)) |
||||
|
||||
def _bootstrap(self) -> None: |
||||
|
||||
if call('openstack', 'user', 'show', 'admin'): |
||||
return |
||||
|
||||
bootstrap_url = 'http://{extgateway}:5000/v3/'.format(**_env) |
||||
|
||||
check('snap-openstack', 'launch', 'keystone-manage', 'bootstrap', |
||||
'--bootstrap-password', _env['ospassword'], |
||||
'--bootstrap-admin-url', bootstrap_url, |
||||
'--bootstrap-internal-url', bootstrap_url, |
||||
'--bootstrap-public-url', bootstrap_url) |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
"""Setup Databases. |
||||
|
||||
Create all the MySQL databases we require, then setup the |
||||
fernet keys and create the service project. |
||||
|
||||
""" |
||||
log.info('Waiting for MySQL server to start ...') |
||||
self._wait() |
||||
log.info('Mysql server started! Creating databases ...') |
||||
self._create_dbs() |
||||
|
||||
log.info('Configuring Keystone Fernet Keys ...') |
||||
check('snap-openstack', 'launch', 'keystone-manage', |
||||
'fernet_setup', '--keystone-user', 'root', |
||||
'--keystone-group', 'root') |
||||
check('snap-openstack', 'launch', 'keystone-manage', 'db_sync') |
||||
|
||||
restart('keystone-*') |
||||
|
||||
log.info('Bootstrapping Keystone ...') |
||||
self._bootstrap() |
||||
|
||||
log.info('Creating service project ...') |
||||
if not call('openstack', 'project', 'show', 'service'): |
||||
check('openstack', 'project', 'create', '--domain', |
||||
'default', '--description', 'Service Project', |
||||
'service') |
||||
|
||||
log.info('Keystone configured!') |
||||
|
||||
|
||||
class NovaSetup(Question): |
||||
"""Create all relevant nova users and services.""" |
||||
|
||||
def _flavors(self) -> None: |
||||
"""Create default flavors.""" |
||||
|
||||
if not call('openstack', 'flavor', 'show', 'm1.tiny'): |
||||
check('openstack', 'flavor', 'create', '--id', '1', |
||||
'--ram', '512', '--disk', '1', '--vcpus', '1', 'm1.tiny') |
||||
if not call('openstack', 'flavor', 'show', 'm1.small'): |
||||
check('openstack', 'flavor', 'create', '--id', '2', |
||||
'--ram', '2048', '--disk', '20', '--vcpus', '1', 'm1.small') |
||||
if not call('openstack', 'flavor', 'show', 'm1.medium'): |
||||
check('openstack', 'flavor', 'create', '--id', '3', |
||||
'--ram', '4096', '--disk', '20', '--vcpus', '2', 'm1.medium') |
||||
if not call('openstack', 'flavor', 'show', 'm1.large'): |
||||
check('openstack', 'flavor', 'create', '--id', '4', |
||||
'--ram', '8192', '--disk', '20', '--vcpus', '4', 'm1.large') |
||||
if not call('openstack', 'flavor', 'show', 'm1.xlarge'): |
||||
check('openstack', 'flavor', 'create', '--id', '5', |
||||
'--ram', '16384', '--disk', '20', '--vcpus', '8', |
||||
'm1.xlarge') |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
log.info('Configuring nova ...') |
||||
|
||||
if not call('openstack', 'user', 'show', 'nova'): |
||||
check('openstack', 'user', 'create', '--domain', |
||||
'default', '--password', 'nova', 'nova') |
||||
check('openstack', 'role', 'add', '--project', |
||||
'service', '--user', 'nova', 'admin') |
||||
|
||||
if not call('openstack', 'user', 'show', 'placement'): |
||||
check('openstack', 'user', 'create', '--domain', 'default', |
||||
'--password', 'placement', 'placement') |
||||
check('openstack', 'role', 'add', '--project', 'service', |
||||
'--user', 'placement', 'admin') |
||||
|
||||
if not call('openstack', 'service', 'show', 'compute'): |
||||
check('openstack', 'service', 'create', '--name', 'nova', |
||||
'--description', '"Openstack Compute"', 'compute') |
||||
for endpoint in ['public', 'internal', 'admin']: |
||||
call('openstack', 'endpoint', 'create', '--region', |
||||
'microstack', 'compute', endpoint, |
||||
'http://{extgateway}:8774/v2.1'.format(**_env)) |
||||
|
||||
if not call('openstack', 'service', 'show', 'placement'): |
||||
check('openstack', 'service', 'create', '--name', |
||||
'placement', '--description', '"Placement API"', |
||||
'placement') |
||||
|
||||
for endpoint in ['public', 'internal', 'admin']: |
||||
call('openstack', 'endpoint', 'create', '--region', |
||||
'microstack', 'placement', endpoint, |
||||
'http://{extgateway}:8778'.format(**_env)) |
||||
|
||||
# Grant nova user access to cell0 |
||||
sql( |
||||
"GRANT ALL PRIVILEGES ON nova_cell0.* TO 'nova'@'{extgateway}' \ |
||||
IDENTIFIED BY \'nova';".format(**_env)) |
||||
|
||||
check('snap-openstack', 'launch', 'nova-manage', 'api_db', 'sync') |
||||
|
||||
if 'cell0' not in check_output('snap-openstack', 'launch', |
||||
'nova-manage', 'cell_v2', |
||||
'list_cells'): |
||||
check('snap-openstack', 'launch', 'nova-manage', |
||||
'cell_v2', 'map_cell0') |
||||
|
||||
if 'cell1' not in check_output('snap-openstack', 'launch', |
||||
'nova-manage', 'cell_v2', 'list_cells'): |
||||
|
||||
check('snap-openstack', 'launch', 'nova-manage', 'cell_v2', |
||||
'create_cell', '--name=cell1', '--verbose') |
||||
|
||||
check('snap-openstack', 'launch', 'nova-manage', 'db', 'sync') |
||||
|
||||
restart('nova-*') |
||||
|
||||
nc_wait(_env['extgateway'], '8774') |
||||
|
||||
sleep(5) # TODO: log_wait |
||||
|
||||
log.info('Creating default flavors...') |
||||
self._flavors() |
||||
|
||||
|
||||
class NeutronSetup(Question): |
||||
"""Create all relevant neutron services and users.""" |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
log.info('Configuring Neutron') |
||||
|
||||
if not call('openstack', 'user', 'show', 'neutron'): |
||||
check('openstack', 'user', 'create', '--domain', 'default', |
||||
'--password', 'neutron', 'neutron') |
||||
check('openstack', 'role', 'add', '--project', 'service', |
||||
'--user', 'neutron', 'admin') |
||||
|
||||
if not call('openstack', 'service', 'show', 'network'): |
||||
check('openstack', 'service', 'create', '--name', 'neutron', |
||||
'--description', '"OpenStack Network"', 'network') |
||||
for endpoint in ['public', 'internal', 'admin']: |
||||
call('openstack', 'endpoint', 'create', '--region', |
||||
'microstack', 'network', endpoint, |
||||
'http://{extgateway}:9696'.format(**_env)) |
||||
|
||||
check('snap-openstack', 'launch', 'neutron-db-manage', 'upgrade', |
||||
'head') |
||||
|
||||
restart('neutron-*') |
||||
|
||||
nc_wait(_env['extgateway'], '9696') |
||||
|
||||
sleep(5) # TODO: log_wait |
||||
|
||||
if not call('openstack', 'network', 'show', 'test'): |
||||
check('openstack', 'network', 'create', 'test') |
||||
|
||||
if not call('openstack', 'subnet', 'show', 'test-subnet'): |
||||
check('openstack', 'subnet', 'create', '--network', 'test', |
||||
'--subnet-range', '192.168.222.0/24', 'test-subnet') |
||||
|
||||
if not call('openstack', 'network', 'show', 'external'): |
||||
check('openstack', 'network', 'create', '--external', |
||||
'--provider-physical-network=physnet1', |
||||
'--provider-network-type=flat', 'external') |
||||
if not call('openstack', 'subnet', 'show', 'external-subnet'): |
||||
check('openstack', 'subnet', 'create', '--network', 'external', |
||||
'--subnet-range', _env['extcidr'], '--no-dhcp', |
||||
'external-subnet') |
||||
|
||||
if not call('openstack', 'router', 'show', 'test-router'): |
||||
check('openstack', 'router', 'create', 'test-router') |
||||
check('openstack', 'router', 'add', 'subnet', 'test-router', |
||||
'test-subnet') |
||||
check('openstack', 'router', 'set', '--external-gateway', |
||||
'external', 'test-router') |
||||
|
||||
|
||||
class GlanceSetup(Question): |
||||
"""Setup glance, and download an initial Cirros image.""" |
||||
|
||||
def _fetch_cirros(self) -> None: |
||||
|
||||
if call('openstack', 'image', 'show', 'cirros'): |
||||
return |
||||
|
||||
env = dict(**_env) |
||||
env['VER'] = '0.4.0' |
||||
env['IMG'] = 'cirros-{VER}-x86_64-disk.img'.format(**env) |
||||
|
||||
log.info('Fetching cirros image ...') |
||||
|
||||
cirros_path = '{SNAP_COMMON}/images/{IMG}'.format(**env) |
||||
|
||||
if not path.exists(cirros_path): |
||||
check('mkdir', '-p', '{SNAP_COMMON}/images'.format(**env)) |
||||
download( |
||||
'http://download.cirros-cloud.net/{VER}/{IMG}'.format(**env), |
||||
'{SNAP_COMMON}/images/{IMG}'.format(**env)) |
||||
|
||||
check('openstack', 'image', 'create', '--file', |
||||
'{SNAP_COMMON}/images/{IMG}'.format(**env), |
||||
'--public', '--container-format=bare', |
||||
'--disk-format=qcow2', 'cirros') |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
|
||||
log.info('Configuring Glance ...') |
||||
|
||||
if not call('openstack', 'user', 'show', 'glance'): |
||||
check('openstack', 'user', 'create', '--domain', 'default', |
||||
'--password', 'glance', 'glance') |
||||
check('openstack', 'role', 'add', '--project', 'service', |
||||
'--user', 'glance', 'admin') |
||||
|
||||
if not call('openstack', 'service', 'show', 'image'): |
||||
check('openstack', 'service', 'create', '--name', 'glance', |
||||
'--description', '"OpenStack Image"', 'image') |
||||
for endpoint in ['internal', 'admin', 'public']: |
||||
check('openstack', 'endpoint', 'create', '--region', |
||||
'microstack', 'image', endpoint, |
||||
'http://{extgateway}:9292'.format(**_env)) |
||||
|
||||
check('snap-openstack', 'launch', 'glance-manage', 'db_sync') |
||||
|
||||
restart('glance*') |
||||
|
||||
nc_wait(_env['extgateway'], '9292') |
||||
|
||||
sleep(5) # TODO: log_wait |
||||
|
||||
self._fetch_cirros() |
||||
|
||||
|
||||
class PostSetup(Question): |
||||
"""Sneak in any additional cleanup, then set the initialized state.""" |
||||
|
||||
def yes(self, answer: str) -> None: |
||||
|
||||
log.info('restarting libvirt and virtlogd ...') |
||||
# This fixes an issue w/ logging not getting set. |
||||
# TODO: fix issue. |
||||
restart('*virt*') |
||||
|
||||
check('snapctl', 'set', 'initialized=true') |
||||
log.info('Complete. Marked microstack as initialized!') |
||||
@ -0,0 +1,136 @@ |
||||
"""shell.py |
||||
|
||||
Contains wrappers around subprocess and pymysql commands, specific to |
||||
our needs in the init script. |
||||
|
||||
# TODO capture stdout (and output to log.DEBUG) |
||||
|
||||
---------------------------------------------------------------------- |
||||
|
||||
Copyright 2019 Canonical Ltd |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
|
||||
""" |
||||
|
||||
import subprocess |
||||
from time import sleep |
||||
from typing import Dict, List |
||||
|
||||
import pymysql |
||||
import wget |
||||
|
||||
from init.config import Env |
||||
|
||||
|
||||
_env = Env().get_env() |
||||
|
||||
|
||||
def check(*args: List[str], env: Dict = _env) -> int: |
||||
"""Execute a shell command, raising an error on failed excution. |
||||
|
||||
:param args: strings to be composed into the bash call. |
||||
:param env: defaults to our Env singleton; can be overriden. |
||||
|
||||
""" |
||||
return subprocess.check_call(args, env=env) |
||||
|
||||
|
||||
def check_output(*args: List[str], env: Dict = _env) -> str: |
||||
"""Execute a shell command, returning the output of the command. |
||||
|
||||
:param args: strings to be composed into the bash call. |
||||
:param env: defaults to our Env singleton; can be overriden. |
||||
|
||||
Include our env; pass in any extra keyword args. |
||||
""" |
||||
return subprocess.check_output(args, env=env, |
||||
universal_newlines=True).strip() |
||||
|
||||
|
||||
def call(*args: List[str], env: Dict = _env) -> bool: |
||||
"""Execute a shell command. |
||||
|
||||
Return True if the call executed successfully (returned 0), or |
||||
False if it returned with an error code (return > 0) |
||||
|
||||
:param args: strings to be composed into the bash call. |
||||
:param env: defaults to our Env singleton; can be overriden. |
||||
""" |
||||
return not subprocess.call(args, env=env) |
||||
|
||||
|
||||
def shell(cmd: str, env: Dict = _env) -> int: |
||||
"""Execute a command, using the actual bourne again shell. |
||||
|
||||
Use this in cases where it is difficult to compose a comma |
||||
separate list that will get parsed into a succesful bash |
||||
command. (E.g., your bash command contains an argument like ".*" |
||||
".*" ".*") |
||||
|
||||
:param cmd: the command to run. |
||||
:param env: defaults to our Env singleton; can be overriden. |
||||
|
||||
""" |
||||
return subprocess.check_call(cmd, shell=True, env=env, |
||||
executable='/snap/core18/current/bin/bash') |
||||
|
||||
|
||||
def sql(cmd: str) -> None: |
||||
"""Execute some SQL! |
||||
|
||||
Really simply wrapper around a pymysql connection, suitable for |
||||
passing the limited CREATE and GRANT commands that we need to pass |
||||
in our init script. |
||||
|
||||
:param cmd: sql to execute. |
||||
|
||||
""" |
||||
mysql_conf = '${SNAP_USER_COMMON}/etc/mysql/my.cnf'.format(**_env) |
||||
connection = pymysql.connect(host='localhost', user='root', |
||||
read_default_file=mysql_conf) |
||||
|
||||
with connection.cursor() as cursor: |
||||
cursor.execute(cmd) |
||||
|
||||
|
||||
def nc_wait(addr: str, port: str) -> None: |
||||
"""Wait for a service to be answering on a port.""" |
||||
print('Waiting for {}:{}'.format(addr, port)) |
||||
while not call('nc', '-z', addr, port): |
||||
sleep(1) |
||||
|
||||
|
||||
def log_wait(log: str, message: str) -> None: |
||||
"""Wait until a message appears in a log.""" |
||||
while True: |
||||
with open(log, 'r') as log_file: |
||||
for line in log_file.readlines(): |
||||
if message in line: |
||||
return |
||||
sleep(1) |
||||
|
||||
|
||||
def restart(service: str) -> None: |
||||
"""Restart a microstack service. |
||||
|
||||
:param service: the service(s) to be restarted. Can contain wild cards. |
||||
e.g. *rabbit* |
||||
|
||||
""" |
||||
check('systemctl', 'restart', 'snap.microstack.{}'.format(service)) |
||||
|
||||
|
||||
def download(url: str, output: str) -> None: |
||||
"""Download a file to a path""" |
||||
wget.download(url, output) |
||||
@ -0,0 +1,2 @@ |
||||
pymysql |
||||
wget |
||||
@ -0,0 +1,13 @@ |
||||
from setuptools import setup, find_packages |
||||
|
||||
setup( |
||||
name="microstack_init", |
||||
description="Optionally interactive init script for Microstack.", |
||||
packages=find_packages(), |
||||
version="0.0.1", |
||||
entry_points={ |
||||
'console_scripts': [ |
||||
'microstack_init = init.main:main', |
||||
], |
||||
}, |
||||
) |
||||
@ -0,0 +1,4 @@ |
||||
pylint |
||||
pep8 |
||||
stestr |
||||
flake8 |
||||
@ -0,0 +1,125 @@ |
||||
import sys |
||||
import os |
||||
import unittest |
||||
|
||||
# TODO: drop in test runner and get rid of this line. |
||||
sys.path.append(os.getcwd()) # noqa |
||||
|
||||
from init.question import (Question, InvalidQuestion, InvalidAnswer, |
||||
AnswerNotImplemented) |
||||
|
||||
|
||||
############################################################################## |
||||
# |
||||
# Test Fixtures |
||||
# |
||||
############################################################################## |
||||
|
||||
|
||||
class InvalidTypeQuestion(Question): |
||||
_type = 'foo' |
||||
|
||||
|
||||
class IncompleteQuestion(Question): |
||||
_type = 'auto' |
||||
|
||||
|
||||
class GoodAutoQuestion(Question): |
||||
_type = 'auto' |
||||
|
||||
def yes(self, answer): |
||||
return 'I am a good question!' |
||||
|
||||
|
||||
class GoodBinaryQuestion(Question): |
||||
_type = 'binary' |
||||
|
||||
def yes(self, answer): |
||||
return True |
||||
|
||||
def no(self, answer): |
||||
return False |
||||
|
||||
|
||||
class GoodStringQuestion(Question): |
||||
"""Pass a string through to the output of Question.ask. |
||||
|
||||
# TODO right now, we have separate handlers for Truthy and Falsey |
||||
answers, and this test class basically makes them do the same |
||||
thing. Is this a good pattern? |
||||
|
||||
""" |
||||
_type = 'string' |
||||
|
||||
def yes(self, answer): |
||||
return answer |
||||
|
||||
def no(self, answer): |
||||
return answer |
||||
|
||||
|
||||
############################################################################## |
||||
# |
||||
# Tests Proper |
||||
# |
||||
############################################################################## |
||||
|
||||
|
||||
class TestQuestionClass(unittest.TestCase): |
||||
""" |
||||
Test basic features of the Question class. |
||||
|
||||
""" |
||||
def test_invalid_type(self): |
||||
|
||||
with self.assertRaises(InvalidQuestion): |
||||
InvalidTypeQuestion().ask() |
||||
|
||||
def test_valid_type(self): |
||||
|
||||
self.assertTrue(GoodBinaryQuestion()) |
||||
|
||||
def test_not_implemented(self): |
||||
|
||||
with self.assertRaises(AnswerNotImplemented): |
||||
IncompleteQuestion().ask() |
||||
|
||||
def test_auto_question(self): |
||||
|
||||
self.assertEqual(GoodAutoQuestion().ask(), 'I am a good question!') |
||||
|
||||
|
||||
class TestInput(unittest.TestCase): |
||||
""" |
||||
Test input handling. |
||||
|
||||
Takes advantage of the fact that we can override the Question |
||||
class's input handler. |
||||
|
||||
""" |
||||
def test_binary_question(self): |
||||
|
||||
q = GoodBinaryQuestion() |
||||
|
||||
for answer in ['yes', 'Yes', 'y']: |
||||
q._input_func = lambda x: answer.encode('utf8') |
||||
self.assertTrue(q.ask()) |
||||
|
||||
for answer in ['No', 'n', 'no']: |
||||
q._input_func = lambda x: answer.encode('utf8') |
||||
self.assertFalse(q.ask()) |
||||
|
||||
with self.assertRaises(InvalidAnswer): |
||||
q._input_func = lambda x: 'foo'.encode('utf8') |
||||
q.ask() |
||||
|
||||
def test_string_question(self): |
||||
q = GoodStringQuestion() |
||||
|
||||
for answer in ['foo', 'bar', 'baz', '', 'yadayadayada']: |
||||
q._input_func = lambda x: answer.encode('utf8') |
||||
self.assertEqual(answer, q.ask()) |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
unittest.main() |
||||