diff --git a/.zuul.yaml b/.zuul.yaml index e79b21a..9a49852 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,7 +1,7 @@ - job: name: microstack-tox-snap-with-sudo parent: openstack-tox-snap-with-sudo - timeout: 7200 + timeout: 5400 nodeset: ubuntu-bionic vars: tox_envlist: snap diff --git a/DEMO.md b/DEMO.md index 2dc6625..6830d3f 100644 --- a/DEMO.md +++ b/DEMO.md @@ -114,10 +114,9 @@ sudo systemctl restart snap.microstack.* Create a test instance in your cloud. -`microstack.launch test` +`microstack.launch cirros --name test` -This will launch a machine using the built-in cirros image, and also -do some other nice things like setting up security groups. Once the +This will launch a machine using the built-in cirros image. Once the machine is setup, verify that you can ping it, then tear it down. ``` @@ -171,7 +170,7 @@ You'll need to load microstack credentials. You can temporarily drop into the microstack snap's shell environment to make this easy. ``` -snap run --shell microstack.launch +snap run --shell microstack.init juju autoload-credentials exit ``` diff --git a/README.md b/README.md index 17b6536..8e37a66 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ To quickly configure networks and launch a vm, run This will configure various Openstack databases. Then run: -`microstack.launch test`. +`microstack.launch cirros --name test`. This will launch an instance for you, and make it available to manage via the command line, or via the Horizon Dashboard. diff --git a/snap-overlay/bin/launch.sh b/snap-overlay/bin/launch.sh index 1fcb835..4c4b62f 100755 --- a/snap-overlay/bin/launch.sh +++ b/snap-overlay/bin/launch.sh @@ -17,29 +17,9 @@ else SERVER=$1 fi -if [[ ! $(openstack keypair list | grep "| microstack |") ]]; then - echo "creating keypair ($HOME/.ssh/id_microstack)" - mkdir -p $HOME/.ssh - chmod 700 $HOME/.ssh - openstack keypair create microstack > $HOME/.ssh/id_microstack - chmod 600 $HOME/.ssh/id_microstack -fi - echo "Launching instance ..." openstack server create --flavor m1.tiny --image cirros --nic net-id=test --key-name microstack $SERVER -echo "Checking security groups ..." -SECGROUP_ID=`openstack security group list --project admin -f value -c ID` -if [[ ! $(openstack security group rule list | grep icmp | grep $SECGROUP_ID) ]]; then - echo "Creating security group rule for ping." - openstack security group rule create $SECGROUP_ID --proto icmp -fi - -if [[ ! $(openstack security group rule list | grep tcp | grep $SECGROUP_ID) ]]; then - echo "Creating security group rule for ssh." - openstack security group rule create $SECGROUP_ID --proto tcp --dst-port 22 -fi - TRIES=0 while [[ $(openstack server list | grep $SERVER | grep ERROR) ]]; do TRIES=$(($TRIES + 1)) diff --git a/snap/hooks/install b/snap/hooks/install index 5edb46d..c74ae91 100755 --- a/snap/hooks/install +++ b/snap/hooks/install @@ -15,6 +15,8 @@ snapctl set \ questions.nova-setup=true \ questions.neutron-setup=true \ questions.glance-setup=true \ + questions.key-pair="id_microstack" \ + questions.security-rules=true \ questions.post-setup=true \ # MySQL snapshot for speedy install diff --git a/snapcraft.yaml b/snapcraft.yaml index 47583a5..7d8367a 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -303,7 +303,7 @@ apps: # Utility to launch a vm. Creates security groups, floating ips, # and other necessities as well. launch: - command: bin/launch.sh + command: microstack_launch # plugs: # - network @@ -820,3 +820,11 @@ parts: requirements: - requirements.txt # Relative to source path, so tools/init/req...txt source: tools/init + + # Launch script + launch: + plugin: python + python-version: python3 + requirements: + - requirements.txt + source: tools/launch diff --git a/tests/test_basic.py b/tests/test_basic.py index 5e0f2a2..ae5c56b 100755 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -56,7 +56,8 @@ class TestBasics(Framework): print("Testing microstack.launch ...") - check(*self.PREFIX, launch, 'breakfast') + check(*self.PREFIX, launch, 'cirros', '--name', 'breakfast', + '--retry') endpoints = check_output( *self.PREFIX, '/snap/bin/microstack.openstack', 'endpoint', 'list') @@ -82,8 +83,8 @@ class TestBasics(Framework): self.assertTrue(ip) pings = 1 - max_pings = 40 - while not call(*self.PREFIX, 'ping', '-c', '1', ip): + max_pings = 600 # ~10 minutes! + while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip): pings += 1 if pings > max_pings: self.assertFalse(True, msg='Max pings reached!') @@ -91,7 +92,7 @@ class TestBasics(Framework): print("Testing instances' ability to connect to the Internet") # Test Internet connectivity attempts = 1 - max_attempts = 40 + max_attempts = 300 # ~10 minutes! username = check_output(*self.PREFIX, 'whoami') while not call( @@ -100,11 +101,11 @@ class TestBasics(Framework): '-oStrictHostKeyChecking=no', '-i', '/home/{}/.ssh/id_microstack'.format(username), 'cirros@{}'.format(ip), - '--', 'ping', '-c', '1', '91.189.94.250'): + '--', 'ping', '-c1', '91.189.94.250'): attempts += 1 if attempts > max_attempts: self.assertFalse(True, msg='Unable to access the Internet!') - time.sleep(5) + time.sleep(1) if 'multipass' in self.PREFIX: print("Opening {}:80 up to the outside world".format( diff --git a/tools/init/init/main.py b/tools/init/init/main.py index 13002da..4089830 100644 --- a/tools/init/init/main.py +++ b/tools/init/init/main.py @@ -59,6 +59,8 @@ def main() -> None: questions.NovaSetup(), questions.NeutronSetup(), questions.GlanceSetup(), + questions.KeyPair(), + questions.SecurityRules(), questions.PostSetup(), ] @@ -68,6 +70,8 @@ def main() -> None: # allow people to pass in a config file from the command line. if CONTROL: check('snapctl', 'set', 'questions.nova-setup=false') + check('snapctl', 'set', 'questions.key-pair=nil') + check('snapctl', 'set', 'questions.security-rules=false') if COMPUTE: check('snapctl', 'set', 'questions.rabbit-mq=false') diff --git a/tools/init/init/question.py b/tools/init/init/question.py index 5f70b74..a742e91 100644 --- a/tools/init/init/question.py +++ b/tools/init/init/question.py @@ -77,7 +77,7 @@ class Question(): raise InvalidQuestion( 'Invalid type {} specified'.format(self._type)) - def _validate(self, answer: bytes) -> Tuple[str, bool]: + def _validate(self, answer: str) -> Tuple[str, bool]: """Validate an answer. :param anwser: raw input from the user. @@ -89,7 +89,9 @@ class Question(): return True, True if self._type == 'string': - # TODO Santize this! + # Allow people to negate a string by passing nil. + if answer.lower() == 'nil': + return None, True return answer, True # self._type is boolean diff --git a/tools/init/init/questions.py b/tools/init/init/questions.py index d1ad7b9..d657bf9 100644 --- a/tools/init/init/questions.py +++ b/tools/init/init/questions.py @@ -23,7 +23,7 @@ limitations under the License. """ - +import json from time import sleep from os import path @@ -532,6 +532,67 @@ class GlanceSetup(Question): self._fetch_cirros() +class KeyPair(Question): + """Create a keypair for ssh access to instances. + + TODO: split the asking from executing of questions, as ask about + this up front. (This needs to run at the end, but for user + experience reasons, we really want to ask all the non auto + questions at the beginning.) + """ + _type = 'string' + + def yes(self, answer: str) -> None: + + if 'microstack' not in check_output('openstack', 'keypair', 'list'): + log.info('Creating microstack keypair (~/.ssh/{})'.format(answer)) + check('mkdir', '-p', '{HOME}/.ssh'.format(**_env)) + check('chmod', '700', '{HOME}/.ssh'.format(**_env)) + id_ = check_output('openstack', 'keypair', 'create', 'microstack') + id_path = '{HOME}/.ssh/{answer}'.format( + HOME=_env['HOME'], + answer=answer + ) + + with open(id_path, 'w') as file_: + file_.write(id_) + check('chmod', '600', id_path) + # TODO: too many assumptions in the below. Make it portable! + user = _env['HOME'].split("/")[2] + check('chown', '{}:{}'.format(user, user), id_path) + + +class SecurityRules(Question): + """Setup default security rules.""" + + _type = 'boolean' + + def yes(self, answer: str) -> None: + # Create security group rules + log.info('Creating security group rules ...') + group_id = check_output('openstack', 'security', 'group', 'list', + '--project', 'admin', '-f', 'value', + '-c', 'ID') + rules = check_output('openstack', 'security', 'group', 'rule', 'list', + '--format', 'json') + ping_rule = False + ssh_rule = False + + for rule in json.loads(rules): + if rule['Security Group'] == group_id: + if rule['IP Protocol'] == 'icmp': + ping_rule = True + if rule['IP Protocol'] == 'tcp': + ssh_rule = True + + if not ping_rule: + check('openstack', 'security', 'group', 'rule', 'create', + group_id, '--proto', 'icmp') + if not ssh_rule: + check('openstack', 'security', 'group', 'rule', 'create', + group_id, '--proto', 'tcp', '--dst-port', '22') + + class PostSetup(Question): """Sneak in any additional cleanup, then set the initialized state.""" diff --git a/tools/launch/launch/__init__.py b/tools/launch/launch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/launch/launch/main.py b/tools/launch/launch/main.py new file mode 100644 index 0000000..3c875fd --- /dev/null +++ b/tools/launch/launch/main.py @@ -0,0 +1,167 @@ +import argparse +import json +import petname +import os +import subprocess +import time +import sys + +from typing import List + + +def check(*args: List[str]) -> int: + """Execute a shell command, raising an error on failed excution. + + :param args: strings to be composed into the bash call. + + """ + return subprocess.check_call(args, env=os.environ) + + +def check_output(*args: List[str]) -> str: + """Execute a shell command, returning the output of the command. + + :param args: strings to be composed into the bash call. + + Include our env; pass in any extra keyword args. + """ + return subprocess.check_output(args, universal_newlines=True, + env=os.environ).strip() + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('image', + help='The name of the openstack image to use.') + parser.add_argument('-n', '--name', help='The name of the instance') + parser.add_argument('-k', '--key', help='ssh key to use', + default='microstack') + parser.add_argument('-f', '--flavor', help='Flavor to use.', + default='m1.tiny') + parser.add_argument('-t', '--net-id', help='Network', default='test') + parser.add_argument('-w', '--wait', action='store_true', + help='Wait for server to become active before exiting') + parser.add_argument('-r', '--retry', action='store_true', + help='Retry failed launch attempts') + + args = parser.parse_args() + return args + + +def create_server(name, args): + + ret = check_output('openstack', 'server', 'create', + '--flavor', args.flavor, + '--image', args.image, + '--nic', 'net-id={}'.format(args.net_id), + '--key-name', args.key, + name, '--format', 'json') + ret = json.loads(ret) + return ret['id'] + + +def delete_server(server_id): + check('openstack', 'server', 'delete', server_id) + + +def check_server(name, server_id, args): + status = 'Unknown' + + retries = 0 + max_retries = 10 + + waits = 0 + max_waits = 1000 # 100 seconds + ~1000 calls to `openstack server list`. + + while True: + status_ = check_output('openstack', 'server', 'list', + '--format', 'json') + status_ = json.loads(status_) + for server in status_: + if server['ID'] == server_id: + status = server['Status'] + + if not status: + # Something went wrong ... + break + + if not args.wait and not args.retry: + # Just return BUILD or ACTIVE or Unknown. + break + + if waits < 1: + print("Waiting for server to build ...") + + if status == 'BUILD': + if waits <= max_waits: + waits += 1 + time.sleep(0.1) + continue + # Looks like we're stuck! Fall through to ERROR check + # below. + status = 'BUILD (stuck)' + + if status in ['ERROR', 'BUILD (stuck)']: + if not args.retry or retries > max_retries: + break + + print('Ran into an error launching server. Retrying ...') + delete_server(server_id) + server_id = create_server(name, args) + waits = 0 # Reset waits + retries += 1 + continue + + if status == 'ACTIVE': + break + + return (status, server_id) + + +def launch(name, args): + """Launch a server!""" + + print("Launching server ...") + server_id = create_server(name, args) + + status, server_id = check_server(name, server_id, args) + if status not in ['BUILD', 'ACTIVE']: + print('Uh-oh. Something went wrong launching {}. Status is {}.'.format( + name, status)) + sys.exit(1) + + print('Allocating floating ip ...') + ip = check_output('openstack', 'floating', 'ip', 'create', '-f', 'value', + '-c', 'floating_ip_address', 'external') + check('openstack', 'server', 'add', 'floating', 'ip', server_id, ip) + + print("""\ +Server {} launched! (status is {}) + +Access it with `ssh -i \ +$HOME/.ssh/id_microstack` @{}""".format(name, status, ip)) + + gate = check_output('snapctl', 'get', 'questions.ext-gateway') + print('You can also visit the OpenStack dashboard at http://{}'.format( + gate)) + + +def main(): + args = parse_args() + name = args.name or petname.generate() + + # Parse microstack.rc + # TODO: we need a share lib that does this in a more robust way. + mstackrc = '{SNAP_COMMON}/etc/microstack.rc'.format(**os.environ) + with open(mstackrc, 'r') as rc_file: + for line in rc_file.readlines(): + if not line.startswith('export'): + continue + key, val = line[7:].split('=') + os.environ[key.strip()] = val.strip() + + return launch(name, args) + + +if __name__ == '__main__': + main() diff --git a/tools/launch/requirements.txt b/tools/launch/requirements.txt new file mode 100644 index 0000000..28ab56a --- /dev/null +++ b/tools/launch/requirements.txt @@ -0,0 +1 @@ +petname diff --git a/tools/launch/setup.py b/tools/launch/setup.py new file mode 100644 index 0000000..fe3a588 --- /dev/null +++ b/tools/launch/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup, find_packages + +setup( + name="microstack_launch", + description="Launch an instance!", + packages=find_packages(exclude=("tests",)), + version="0.0.1", + entry_points={ + 'console_scripts': [ + 'microstack_launch = launch.main:main', + ], + }, +) diff --git a/tox.ini b/tox.ini index 37b3193..90d6a9d 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ commands = # Specify tests in sequence, as they can't run in parallel if not # using multipass. {toxinidir}/tests/test_basic.py - {toxinidir}/tests/test_control.py + {toxinidir}/tests/test_control.py [testenv:multipass] # Default testing environment for a human operated machine. Builds the @@ -42,7 +42,7 @@ commands = {toxinidir}/tools/multipass_build.sh flake8 {toxinidir}/tests/ {toxinidir}/tests/test_basic.py - {toxinidir}/tests/test_control.py + {toxinidir}/tests/test_control.py [testenv:basic] # Just run basic_test.sh, with multipass support. @@ -54,7 +54,14 @@ commands = {toxinidir}/tools/basic_setup.sh flake8 {toxinidir}/tests/ {toxinidir}/tests/test_basic.py - {toxinidir}/tests/test_control.py + {toxinidir}/tests/test_control.py + +[testenv:lint] +deps = -r{toxinidir}/test-requirements.txt +commands = + flake8 {toxinidir}/tests/ + flake8 {toxinidir}/tools/init/init/ + flake8 {toxinidir}/tools/launch/launch/ [testenv:init_lint] deps = -r{toxinidir}/tools/init/test-requirements.txt