Browse Source

Ported and updated launch script

Moved security rules and keypair creation into init first.

Launch script now takes image name as positional argument, and name of
instance as a named argument. This makes it work more like launch in
other Canonical tools.

Written in Python, for ease of maintenance.

--retry and --wait args allow it to behave like tests expect it to,
while humans will get a much more intuitive (and much less noisy)
experience.

Also increased time we wait for a ping on the host, to allow for
slower, pure qemu, emulation times, and bring it in line with what
Tempest does in similar situations.

Change-Id: I11dcc098012468e9c88dcc7af78cde6920f31ecd
changes/08/688008/13
Pete Vander Giessen 1 month ago
parent
commit
0399955cf1
15 changed files with 284 additions and 39 deletions
  1. +1
    -1
      .zuul.yaml
  2. +3
    -4
      DEMO.md
  3. +1
    -1
      README.md
  4. +0
    -20
      snap-overlay/bin/launch.sh
  5. +2
    -0
      snap/hooks/install
  6. +9
    -1
      snapcraft.yaml
  7. +7
    -6
      tests/test_basic.py
  8. +4
    -0
      tools/init/init/main.py
  9. +4
    -2
      tools/init/init/question.py
  10. +62
    -1
      tools/init/init/questions.py
  11. +0
    -0
      tools/launch/launch/__init__.py
  12. +167
    -0
      tools/launch/launch/main.py
  13. +1
    -0
      tools/launch/requirements.txt
  14. +13
    -0
      tools/launch/setup.py
  15. +10
    -3
      tox.ini

+ 1
- 1
.zuul.yaml View File

@@ -1,7 +1,7 @@
- job:
name: microstack-tox-snap-with-sudo
parent: openstack-tox-snap-with-sudo
timeout: 7200
timeout: 5400
nodeset: ubuntu-bionic
vars:
tox_envlist: snap

+ 3
- 4
DEMO.md View File

@@ -114,10 +114,9 @@ sudo systemctl restart snap.microstack.*

Create a test instance in your cloud.

`microstack.launch test`
`microstack.launch cirros --name test`

This will launch a machine using the built-in cirros image, and also
do some other nice things like setting up security groups. Once the
This will launch a machine using the built-in cirros image. Once the
machine is setup, verify that you can ping it, then tear it down.

```
@@ -171,7 +170,7 @@ You'll need to load microstack credentials. You can temporarily drop
into the microstack snap's shell environment to make this easy.

```
snap run --shell microstack.launch
snap run --shell microstack.init
juju autoload-credentials
exit
```

+ 1
- 1
README.md View File

@@ -28,7 +28,7 @@ To quickly configure networks and launch a vm, run

This will configure various Openstack databases. Then run:

`microstack.launch test`.
`microstack.launch cirros --name test`.

This will launch an instance for you, and make it available to manage via the command line, or via the Horizon Dashboard.


+ 0
- 20
snap-overlay/bin/launch.sh View File

@@ -17,29 +17,9 @@ else
SERVER=$1
fi

if [[ ! $(openstack keypair list | grep "| microstack |") ]]; then
echo "creating keypair ($HOME/.ssh/id_microstack)"
mkdir -p $HOME/.ssh
chmod 700 $HOME/.ssh
openstack keypair create microstack > $HOME/.ssh/id_microstack
chmod 600 $HOME/.ssh/id_microstack
fi

echo "Launching instance ..."
openstack server create --flavor m1.tiny --image cirros --nic net-id=test --key-name microstack $SERVER

echo "Checking security groups ..."
SECGROUP_ID=`openstack security group list --project admin -f value -c ID`
if [[ ! $(openstack security group rule list | grep icmp | grep $SECGROUP_ID) ]]; then
echo "Creating security group rule for ping."
openstack security group rule create $SECGROUP_ID --proto icmp
fi

if [[ ! $(openstack security group rule list | grep tcp | grep $SECGROUP_ID) ]]; then
echo "Creating security group rule for ssh."
openstack security group rule create $SECGROUP_ID --proto tcp --dst-port 22
fi

TRIES=0
while [[ $(openstack server list | grep $SERVER | grep ERROR) ]]; do
TRIES=$(($TRIES + 1))

+ 2
- 0
snap/hooks/install View File

@@ -15,6 +15,8 @@ snapctl set \
questions.nova-setup=true \
questions.neutron-setup=true \
questions.glance-setup=true \
questions.key-pair="id_microstack" \
questions.security-rules=true \
questions.post-setup=true \

# MySQL snapshot for speedy install

+ 9
- 1
snapcraft.yaml View File

@@ -303,7 +303,7 @@ apps:
# Utility to launch a vm. Creates security groups, floating ips,
# and other necessities as well.
launch:
command: bin/launch.sh
command: microstack_launch
# plugs:
# - network

@@ -820,3 +820,11 @@ parts:
requirements:
- requirements.txt # Relative to source path, so tools/init/req...txt
source: tools/init

# Launch script
launch:
plugin: python
python-version: python3
requirements:
- requirements.txt
source: tools/launch

+ 7
- 6
tests/test_basic.py View File

@@ -56,7 +56,8 @@ class TestBasics(Framework):

print("Testing microstack.launch ...")

check(*self.PREFIX, launch, 'breakfast')
check(*self.PREFIX, launch, 'cirros', '--name', 'breakfast',
'--retry')

endpoints = check_output(
*self.PREFIX, '/snap/bin/microstack.openstack', 'endpoint', 'list')
@@ -82,8 +83,8 @@ class TestBasics(Framework):
self.assertTrue(ip)

pings = 1
max_pings = 40
while not call(*self.PREFIX, 'ping', '-c', '1', ip):
max_pings = 600 # ~10 minutes!
while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
pings += 1
if pings > max_pings:
self.assertFalse(True, msg='Max pings reached!')
@@ -91,7 +92,7 @@ class TestBasics(Framework):
print("Testing instances' ability to connect to the Internet")
# Test Internet connectivity
attempts = 1
max_attempts = 40
max_attempts = 300 # ~10 minutes!
username = check_output(*self.PREFIX, 'whoami')

while not call(
@@ -100,11 +101,11 @@ class TestBasics(Framework):
'-oStrictHostKeyChecking=no',
'-i', '/home/{}/.ssh/id_microstack'.format(username),
'cirros@{}'.format(ip),
'--', 'ping', '-c', '1', '91.189.94.250'):
'--', 'ping', '-c1', '91.189.94.250'):
attempts += 1
if attempts > max_attempts:
self.assertFalse(True, msg='Unable to access the Internet!')
time.sleep(5)
time.sleep(1)

if 'multipass' in self.PREFIX:
print("Opening {}:80 up to the outside world".format(

+ 4
- 0
tools/init/init/main.py View File

@@ -59,6 +59,8 @@ def main() -> None:
questions.NovaSetup(),
questions.NeutronSetup(),
questions.GlanceSetup(),
questions.KeyPair(),
questions.SecurityRules(),
questions.PostSetup(),
]

@@ -68,6 +70,8 @@ def main() -> None:
# allow people to pass in a config file from the command line.
if CONTROL:
check('snapctl', 'set', 'questions.nova-setup=false')
check('snapctl', 'set', 'questions.key-pair=nil')
check('snapctl', 'set', 'questions.security-rules=false')

if COMPUTE:
check('snapctl', 'set', 'questions.rabbit-mq=false')

+ 4
- 2
tools/init/init/question.py View File

@@ -77,7 +77,7 @@ class Question():
raise InvalidQuestion(
'Invalid type {} specified'.format(self._type))

def _validate(self, answer: bytes) -> Tuple[str, bool]:
def _validate(self, answer: str) -> Tuple[str, bool]:
"""Validate an answer.

:param anwser: raw input from the user.
@@ -89,7 +89,9 @@ class Question():
return True, True

if self._type == 'string':
# TODO Santize this!
# Allow people to negate a string by passing nil.
if answer.lower() == 'nil':
return None, True
return answer, True

# self._type is boolean

+ 62
- 1
tools/init/init/questions.py View File

@@ -23,7 +23,7 @@ limitations under the License.

"""

import json
from time import sleep
from os import path

@@ -532,6 +532,67 @@ class GlanceSetup(Question):
self._fetch_cirros()


class KeyPair(Question):
"""Create a keypair for ssh access to instances.

TODO: split the asking from executing of questions, as ask about
this up front. (This needs to run at the end, but for user
experience reasons, we really want to ask all the non auto
questions at the beginning.)
"""
_type = 'string'

def yes(self, answer: str) -> None:

if 'microstack' not in check_output('openstack', 'keypair', 'list'):
log.info('Creating microstack keypair (~/.ssh/{})'.format(answer))
check('mkdir', '-p', '{HOME}/.ssh'.format(**_env))
check('chmod', '700', '{HOME}/.ssh'.format(**_env))
id_ = check_output('openstack', 'keypair', 'create', 'microstack')
id_path = '{HOME}/.ssh/{answer}'.format(
HOME=_env['HOME'],
answer=answer
)

with open(id_path, 'w') as file_:
file_.write(id_)
check('chmod', '600', id_path)
# TODO: too many assumptions in the below. Make it portable!
user = _env['HOME'].split("/")[2]
check('chown', '{}:{}'.format(user, user), id_path)


class SecurityRules(Question):
"""Setup default security rules."""

_type = 'boolean'

def yes(self, answer: str) -> None:
# Create security group rules
log.info('Creating security group rules ...')
group_id = check_output('openstack', 'security', 'group', 'list',
'--project', 'admin', '-f', 'value',
'-c', 'ID')
rules = check_output('openstack', 'security', 'group', 'rule', 'list',
'--format', 'json')
ping_rule = False
ssh_rule = False

for rule in json.loads(rules):
if rule['Security Group'] == group_id:
if rule['IP Protocol'] == 'icmp':
ping_rule = True
if rule['IP Protocol'] == 'tcp':
ssh_rule = True

if not ping_rule:
check('openstack', 'security', 'group', 'rule', 'create',
group_id, '--proto', 'icmp')
if not ssh_rule:
check('openstack', 'security', 'group', 'rule', 'create',
group_id, '--proto', 'tcp', '--dst-port', '22')


class PostSetup(Question):
"""Sneak in any additional cleanup, then set the initialized state."""


+ 0
- 0
tools/launch/launch/__init__.py View File


+ 167
- 0
tools/launch/launch/main.py View File

@@ -0,0 +1,167 @@
import argparse
import json
import petname
import os
import subprocess
import time
import sys

from typing import List


def check(*args: List[str]) -> int:
"""Execute a shell command, raising an error on failed excution.

:param args: strings to be composed into the bash call.

"""
return subprocess.check_call(args, env=os.environ)


def check_output(*args: List[str]) -> str:
"""Execute a shell command, returning the output of the command.

:param args: strings to be composed into the bash call.

Include our env; pass in any extra keyword args.
"""
return subprocess.check_output(args, universal_newlines=True,
env=os.environ).strip()


def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('image',
help='The name of the openstack image to use.')
parser.add_argument('-n', '--name', help='The name of the instance')
parser.add_argument('-k', '--key', help='ssh key to use',
default='microstack')
parser.add_argument('-f', '--flavor', help='Flavor to use.',
default='m1.tiny')
parser.add_argument('-t', '--net-id', help='Network', default='test')
parser.add_argument('-w', '--wait', action='store_true',
help='Wait for server to become active before exiting')
parser.add_argument('-r', '--retry', action='store_true',
help='Retry failed launch attempts')

args = parser.parse_args()
return args


def create_server(name, args):

ret = check_output('openstack', 'server', 'create',
'--flavor', args.flavor,
'--image', args.image,
'--nic', 'net-id={}'.format(args.net_id),
'--key-name', args.key,
name, '--format', 'json')
ret = json.loads(ret)
return ret['id']


def delete_server(server_id):
check('openstack', 'server', 'delete', server_id)


def check_server(name, server_id, args):
status = 'Unknown'

retries = 0
max_retries = 10

waits = 0
max_waits = 1000 # 100 seconds + ~1000 calls to `openstack server list`.

while True:
status_ = check_output('openstack', 'server', 'list',
'--format', 'json')
status_ = json.loads(status_)
for server in status_:
if server['ID'] == server_id:
status = server['Status']

if not status:
# Something went wrong ...
break

if not args.wait and not args.retry:
# Just return BUILD or ACTIVE or Unknown.
break

if waits < 1:
print("Waiting for server to build ...")

if status == 'BUILD':
if waits <= max_waits:
waits += 1
time.sleep(0.1)
continue
# Looks like we're stuck! Fall through to ERROR check
# below.
status = 'BUILD (stuck)'

if status in ['ERROR', 'BUILD (stuck)']:
if not args.retry or retries > max_retries:
break

print('Ran into an error launching server. Retrying ...')
delete_server(server_id)
server_id = create_server(name, args)
waits = 0 # Reset waits
retries += 1
continue

if status == 'ACTIVE':
break

return (status, server_id)


def launch(name, args):
"""Launch a server!"""

print("Launching server ...")
server_id = create_server(name, args)

status, server_id = check_server(name, server_id, args)
if status not in ['BUILD', 'ACTIVE']:
print('Uh-oh. Something went wrong launching {}. Status is {}.'.format(
name, status))
sys.exit(1)

print('Allocating floating ip ...')
ip = check_output('openstack', 'floating', 'ip', 'create', '-f', 'value',
'-c', 'floating_ip_address', 'external')
check('openstack', 'server', 'add', 'floating', 'ip', server_id, ip)

print("""\
Server {} launched! (status is {})

Access it with `ssh -i \
$HOME/.ssh/id_microstack` <username>@{}""".format(name, status, ip))

gate = check_output('snapctl', 'get', 'questions.ext-gateway')
print('You can also visit the OpenStack dashboard at http://{}'.format(
gate))


def main():
args = parse_args()
name = args.name or petname.generate()

# Parse microstack.rc
# TODO: we need a share lib that does this in a more robust way.
mstackrc = '{SNAP_COMMON}/etc/microstack.rc'.format(**os.environ)
with open(mstackrc, 'r') as rc_file:
for line in rc_file.readlines():
if not line.startswith('export'):
continue
key, val = line[7:].split('=')
os.environ[key.strip()] = val.strip()

return launch(name, args)


if __name__ == '__main__':
main()

+ 1
- 0
tools/launch/requirements.txt View File

@@ -0,0 +1 @@
petname

+ 13
- 0
tools/launch/setup.py View File

@@ -0,0 +1,13 @@
from setuptools import setup, find_packages

setup(
name="microstack_launch",
description="Launch an instance!",
packages=find_packages(exclude=("tests",)),
version="0.0.1",
entry_points={
'console_scripts': [
'microstack_launch = launch.main:main',
],
},
)

+ 10
- 3
tox.ini View File

@@ -24,7 +24,7 @@ commands =
# Specify tests in sequence, as they can't run in parallel if not
# using multipass.
{toxinidir}/tests/test_basic.py
{toxinidir}/tests/test_control.py
{toxinidir}/tests/test_control.py

[testenv:multipass]
# Default testing environment for a human operated machine. Builds the
@@ -42,7 +42,7 @@ commands =
{toxinidir}/tools/multipass_build.sh
flake8 {toxinidir}/tests/
{toxinidir}/tests/test_basic.py
{toxinidir}/tests/test_control.py
{toxinidir}/tests/test_control.py

[testenv:basic]
# Just run basic_test.sh, with multipass support.
@@ -54,7 +54,14 @@ commands =
{toxinidir}/tools/basic_setup.sh
flake8 {toxinidir}/tests/
{toxinidir}/tests/test_basic.py
{toxinidir}/tests/test_control.py
{toxinidir}/tests/test_control.py

[testenv:lint]
deps = -r{toxinidir}/test-requirements.txt
commands =
flake8 {toxinidir}/tests/
flake8 {toxinidir}/tools/init/init/
flake8 {toxinidir}/tools/launch/launch/

[testenv:init_lint]
deps = -r{toxinidir}/tools/init/test-requirements.txt

Loading…
Cancel
Save