Clustering prototype

This enables basic clustering functionality. We add:

tools/cluster/cluster/daemon.py: A server that handles validation of
cluster passwords.

tools/cluster/cluster/client.py: A client for this server.

Important Note: This prototype does not support TLS, and the
functionality in the client and server is basic. Before we roll
clustering out to production, we need to have those two chat over TLS,
and be much more careful about verifying credentials.

Also included ...

Various fixes and changes to the init script and config templates to
support cluster configuration, and allow for the fact that we may have
endpoint references for two network ips.

Updates to snapcraft.yaml, adding the new tooling.

A more formalized config infrastructure. It's still a TODO to move the
specification out of the implicit definition in the install hook, and
into a nice, explicit, well documented yaml file.

Added nesting to the Question classes in the init script, as well as
strings pointing at config keys, rather than having the config be
implicitly indicated by the Question subclass' name. (This allows us
to put together a config spec that doesn't require the person reading
the spec to understand what Questions are, and how they are
implemented.)

Renamed and unified the "unit" and "lint" tox environments, to allow
for the multiple Python tools that we want to lint and test.

Added hooks in the init script to make it possible to do automated
testing, and added an automated test for a cluster. Run with "tox -e
cluster".

Added cirros image to snap, to work around sporadic issues downloading
it from download.cirros.net.

Removed ping logic from snap, to workaround failures in gate. Need to
add it back in once we fix them.

Change-Id: I44ccd16168a7ed41486464df8c9e22a14d71ccfd

README.md

@@ -123,3 +123,40 @@ sudo snap enable microstack
 ## Raising a Bug
 
 Please report bugs to the microstack project on launchpad: https://bugs.launchpad.net/microstack
+
+## Clustering Preview
+
+The latests --edge version of the clustering snap contains a preview of microstack's clustering functionality. If you're interested in building a small "edge" cloud with microstack, please take a look at the notes below. Keep in mind that this is preview functionality. Interfaces may not be stable, and the security of the preview is light, and not suitable for production use!
+
+To setup a cluster, you first must setup a control node. Do so with the following commands:
+
+```
+sudo snap install microstack
+sudo microstack.init
+```
+
+Answer the questions in the interactive prompt as follows:
+
+```
+Clustering: yes
+Role: control
+IP Address: Note and accept the default
+```
+
+On a second machine, run:
+
+```
+sudo snap install microstack
+sudo microstack.init
+```
+
+Answer the questions in the interactive prompt as follows:
+
+```
+Setup clustering: yes
+Role: compute
+Control IP: the ip address noted above
+Compute IP: accept the default
+```
+
+You should now have a small, two node cloud, with the first node serving as both the control plane and a hypvervisor, and the second node serving as a hypervisor. You can create vms on both.

snap-overlay/bin/setup-br-ex

@@ -9,7 +9,7 @@
 
 set -ex
 
-extcidr=$(snapctl get questions.ext-cidr)
+extcidr=$(snapctl get config.network.ext-cidr)
 
 # Create external integration bridge
 ovs-vsctl --retry --may-exist add-br br-ex

snap-overlay/snap-openstack.yaml

@@ -52,15 +52,22 @@ setup:
     neutron.keystone.conf.j2: "{snap_common}/etc/neutron/neutron.conf.d/keystone.conf"
     neutron.nova.conf.j2: "{snap_common}/etc/neutron/neutron.conf.d/nova.conf"
     neutron.database.conf.j2: "{snap_common}/etc/neutron/neutron.conf.d/database.conf"
+    neutron.conf.d.rabbitmq.conf.j2: "{snap_common}/etc/neutron/neutron.conf.d/rabbitmq.conf"
 
   chmod:
     "{snap_common}/instances": 0755
     "{snap_common}/etc/microstack.rc": 0644
   snap-config-keys:
-    ospassword: 'questions.os-password'
-    extgateway: 'questions.ext-gateway'
-    extcidr: 'questions.ext-cidr'
-    dns: 'questions.dns'
+    ospassword: 'config.credentials.os-password'
+    nova_password: 'config.credentials.nova-password'
+    neutron_password: 'config.credentials.neutron-password'
+    placement_password: 'config.credentials.placement-password'
+    glance_password: 'config.credentials.glance-password'
+    control_ip: 'config.network.control-ip'
+    compute_ip: 'config.network.compute-ip'
+    extgateway: 'config.network.ext-gateway'
+    extcidr: 'config.network.ext-cidr'
+    dns: 'config.network.dns'
 entry_points:
   keystone-manage:
     binary: "{snap}/bin/keystone-manage"

snap-overlay/templates/clustering-nginx.conf

@@ -0,0 +1,10 @@
+server {
+    listen 8011;
+    error_log syslog:server=unix:/dev/log;
+    access_log syslog:server=unix:/dev/log;
+    location / {
+        include {{ snap }}/usr/conf/uwsgi_params;
+        uwsgi_param SCRIPT_NAME '';
+        uwsgi_pass unix://{{ snap_common }}/run/keystone-api.sock;
+    }
+}

snap-overlay/templates/glance.conf.d.keystone.conf.j2

@@ -1,13 +1,13 @@
 [keystone_authtoken]
-auth_uri = http://{{ extgateway }}:5000
-auth_url = http://{{ extgateway }}:5000
-memcached_servers = {{ extgateway }}:11211
+auth_uri = http://{{ control_ip }}:5000
+auth_url = http://{{ control_ip }}:5000
+memcached_servers = {{ control_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default
 project_name = service
 username = glance
-password = glance
+password = {{ glance_password }}
 
 [paste_deploy]
 flavor = keystone

snap-overlay/templates/glance.database.conf.j2

@@ -1,2 +1,2 @@
 [database]
-connection = mysql+pymysql://glance:glance@{{ extgateway }}/glance
+connection = mysql+pymysql://glance:glance@{{ control_ip }}/glance

snap-overlay/templates/keystone.database.conf.j2

@@ -1,2 +1,2 @@
 [database]
-connection = mysql+pymysql://keystone:keystone@{{ extgateway }}/keystone
+connection = mysql+pymysql://keystone:keystone@{{ control_ip }}/keystone

snap-overlay/templates/microstack.rc.j2

@@ -3,7 +3,7 @@ export OS_USER_DOMAIN_NAME=default
 export OS_PROJECT_NAME=admin
 export OS_USERNAME=admin
 export OS_PASSWORD={{ ospassword }}
-export OS_AUTH_URL=http://{{ extgateway }}:5000
+export OS_AUTH_URL=http://{{ control_ip }}:5000
 export OS_IDENTITY_API_VERSION=3
 export OS_IMAGE_API_VERSION=2
 

snap-overlay/templates/neutron.conf.d.rabbitmq.conf.j2

@@ -0,0 +1,2 @@
+[DEFAULT]
+transport_url = rabbit://openstack:rabbitmq@{{ control_ip }}

snap-overlay/templates/neutron.database.conf.j2

@@ -1,2 +1,2 @@
 [database]
-connection = mysql+pymysql://neutron:neutron@{{ extgateway }}/neutron
+connection = mysql+pymysql://neutron:neutron@{{ control_ip }}/neutron

snap-overlay/templates/neutron.keystone.conf.j2

@@ -2,12 +2,12 @@
 auth_strategy = keystone
 
 [keystone_authtoken]
-auth_uri = http://{{ extgateway }}:5000
-auth_url = http://{{ extgateway }}:5000
-memcached_servers = {{ extgateway }}:11211
+auth_uri = http://{{ control_ip }}:5000
+auth_url = http://{{ control_ip }}:5000
+memcached_servers = {{ control_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default
 project_name = service
 username = neutron
-password = neutron
+password = {{ neutron_password }}

snap-overlay/templates/neutron.nova.conf.j2

@@ -3,11 +3,11 @@ notify_nova_on_port_status_changes = True
 notify_nova_on_port_data_changes = True
 
 [nova]
-auth_url = http://{{ extgateway }}:5000
+auth_url = http://{{ control_ip }}:5000
 auth_type = password
 project_domain_name = default
 user_domain_name = default
 region_name = microstack
 project_name = service
 username = nova
-password = nova
+password = {{ nova_password }}

snap-overlay/templates/nova.conf.d.database.conf.j2

@@ -1,5 +1,5 @@
 [database]
-connection = mysql+pymysql://nova:nova@{{ extgateway }}/nova
+connection = mysql+pymysql://nova:nova@{{ control_ip }}/nova
 
 [api_database]
-connection = mysql+pymysql://nova_api:nova_api@{{ extgateway }}/nova_api
+connection = mysql+pymysql://nova_api:nova_api@{{ control_ip }}/nova_api

snap-overlay/templates/nova.conf.d.glance.conf.j2

@@ -1,2 +1,2 @@
 [glance]
-api_servers = http://{{ extgateway }}:9292
+api_servers = http://{{ control_ip }}:9292

snap-overlay/templates/nova.conf.d.keystone.conf.j2

@@ -1,13 +1,13 @@
 [keystone_authtoken]
-auth_uri = http://{{ extgateway }}:5000
-auth_url = http://{{ extgateway }}:5000
-memcached_servers = {{ extgateway }}:11211
+auth_uri = http://{{ control_ip }}:5000
+auth_url = http://{{ control_ip }}:5000
+memcached_servers = {{ control_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default
 project_name = service
 username = nova
-password = nova
+password = {{ nova_password }}
 
 [paste_deploy]
 flavor = keystone

snap-overlay/templates/nova.conf.d.neutron.conf.j2

@@ -1,13 +1,13 @@
 [neutron]
-url = http://{{ extgateway }}:9696
-auth_url = http://{{ extgateway }}:5000
-memcached_servers = {{ extgateway }}:11211
+url = http://{{ control_ip }}:9696
+auth_url = http://{{ control_ip }}:5000
+memcached_servers = {{ control_ip }}:11211
 auth_type = password
 project_domain_name = default
 user_domain_name = default
 region_name = microstack
 project_name = service
 username = neutron
-password = neutron
+password = {{ neutron_password }}
 service_metadata_proxy = True
 metadata_proxy_shared_secret = supersecret

snap-overlay/templates/nova.conf.d.nova-placement.conf.j2

@@ -4,6 +4,6 @@ project_domain_name = default
 project_name = service
 auth_type = password
 user_domain_name = default
-auth_url = http://{{ extgateway }}:5000
+auth_url = http://{{ control_ip }}:5000
 username = placement
-password = placement
+password = {{ placement_password }}

snap-overlay/templates/nova.conf.d.rabbitmq.conf.j2

@@ -1,2 +1,2 @@
 [DEFAULT]
-transport_url = rabbit://openstack:rabbitmq@{{ extgateway }}
+transport_url = rabbit://openstack:rabbitmq@{{ control_ip }}

snap/hooks/install

@@ -1,29 +1,62 @@
 #!/bin/bash
 set -ex
 
-# Config
-# Set default answers to the questions that microstack.init asks.
+# Initialize Config
 # TODO: put this in a nice yaml format, and parse it.
+
+# General snap-wide settings
+snapctl set \
+        config.clustered=false \
+        config.post-setup=true \
+        ;
+
+# Networking related settings.
+snapctl set \
+        config.network.dns=1.1.1.1 \
+        config.network.ext-gateway=10.20.20.1 \
+        config.network.control-ip=10.20.20.1 \
+        config.network.compute-ip=10.20.20.1 \
+        config.network.ext-cidr=10.20.20.1/24 \
+        config.network.security-rules=true \
+        ;
+
+# Passwords, certs, etc.
+snapctl set \
+        config.credentials.os-password=keystone \
+        config.credentials.key-pair=id_microstack \
+        config.credentials.nova-password=nova \
+        config.credentials.neutron-password=neutron \
+        config.credentials.placement-password=placement \
+        config.credentials.glance-password=glance \
+        ;
+
+# Host optimizations and fixes.
+snapctl set \
+        config.host.ip-forwarding=true \
+        config.host.check-qemu=true \
+        ;
+
+# Enable or disable groups of services.
 snapctl set \
-        questions.ip-forwarding=true \
-        questions.dns=1.1.1.1 \
-        questions.ext-gateway=10.20.20.1 \
-        questions.ext-cidr=10.20.20.1/24 \
-        questions.os-password=keystone \
-        questions.rabbit-mq=true \
-        questions.database-setup=true \
-        questions.nova-setup=true \
-        questions.neutron-setup=true \
-        questions.glance-setup=true \
-        questions.key-pair="id_microstack" \
-        questions.security-rules=true \
-        questions.post-setup=true \
+        config.services.control-plane=true \
+        config.services.hypervisor=true \
+        ;
+
+# Clustering roles
+snapctl set \
+        cluster.role=control \
+        cluster.password=null \
+        ;
 
 # MySQL snapshot for speedy install
 # snapshot is a mysql data dir with
 # rocky keystone,nova,glance,neutron dbs.
 mkdir -p ${SNAP_COMMON}/lib
 
+# Put cirros (and potentially other) images in a user writeable place.
+mkdir -p ${SNAP_COMMON}/images
+cp ${SNAP}/images/* ${SNAP_COMMON}/images/
+
 # Install conf.d configuration from snap for db etc
 echo "Installing configuration for OpenStack Services"
 for project in neutron nova keystone glance; do

snapcraft.yaml

@@ -307,6 +307,19 @@ apps:
 #    plugs:
 #      - network
 
+  # Cluster
+  cluster-server:
+    command: flask run -p 10002 --host=0.0.0.0 # TODO: run as a uwsgi app
+    daemon: simple
+    environment:
+      LC_ALL: C.UTF-8  # Makes flask happy
+      LANG: C.UTF-8  # Makes flask happy
+      FLASK_APP: ${SNAP}/lib/python3.6/site-packages/cluster/daemon.py
+
+  join:
+    command: python3 ${SNAP}/lib/python3.6/site-packages/cluster/client.py
+
+
 parts:
   # Add Ubuntu Cloud Archive sources.
   # Allows us to fetch things such as updated libvirt.
@@ -828,3 +841,11 @@ parts:
     requirements:
       - requirements.txt
     source: tools/launch
+
+  # Clustering client and server
+  cluster:
+    plugin: python
+    python-version: python3
+    requirements:
+      - requirements.txt
+    source: tools/cluster

tests/test_basic.py

@@ -14,10 +14,8 @@ Web IDE.
 
 """
 
-import json
 import os
 import sys
-import time
 import unittest
 import xvfbwrapper
 from selenium import webdriver
@@ -52,7 +50,7 @@ class TestBasics(Framework):
 
         """
         launch = '/snap/bin/microstack.launch'
-        openstack = '/snap/bin/microstack.openstack'
+        # openstack = '/snap/bin/microstack.openstack'
 
         print("Testing microstack.launch ...")
 
@@ -68,45 +66,6 @@ class TestBasics(Framework):
         # Endpoints should not contain localhost
         self.assertFalse("localhost" in endpoints)
 
-        # Verify that microstack.launch completed successfully
-
-        # Ping the instance
-        ip = None
-        servers = check_output(*self.PREFIX, openstack,
-                               'server', 'list', '--format', 'json')
-        servers = json.loads(servers)
-        for server in servers:
-            if server['Name'] == 'breakfast':
-                ip = server['Networks'].split(",")[1].strip()
-                break
-
-        self.assertTrue(ip)
-
-        pings = 1
-        max_pings = 600  # ~10 minutes!
-        while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
-            pings += 1
-            if pings > max_pings:
-                self.assertFalse(True, msg='Max pings reached!')
-
-        print("Testing instances' ability to connect to the Internet")
-        # Test Internet connectivity
-        attempts = 1
-        max_attempts = 300  # ~10 minutes!
-        username = check_output(*self.PREFIX, 'whoami')
-
-        while not call(
-                *self.PREFIX,
-                'ssh',
-                '-oStrictHostKeyChecking=no',
-                '-i', '/home/{}/.ssh/id_microstack'.format(username),
-                'cirros@{}'.format(ip),
-                '--', 'ping', '-c1', '91.189.94.250'):
-            attempts += 1
-            if attempts > max_attempts:
-                self.assertFalse(True, msg='Unable to access the Internet!')
-            time.sleep(1)
-
         if 'multipass' in self.PREFIX:
             print("Opening {}:80 up to the outside world".format(
                 self.HORIZON_IP))

tests/test_cluster.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+"""
+cluster_test.py
+
+This is a test to verify that we can setup a small, two node cluster.
+
+The host running this test must have at least 16GB of RAM, four cpu
+cores, a large amount of disk space, and the ability to run multipass
+vms.
+
+"""
+
+import json
+import os
+import petname
+import sys
+import unittest
+
+sys.path.append(os.getcwd())
+
+from tests.framework import Framework, check, check_output, call  # noqa E402
+
+
+os.environ['MULTIPASS'] = 'true'  # TODO better way to do this.
+
+
+class TestCluster(Framework):
+
+    INIT_FLAG = 'control'
+
+    def _compute_node(self, channel='dangerous'):
+        """Make a compute node.
+
+        TODO: refactor framework so that we can fold a lot of this
+        into the parent framework. There's a lot of dupe code here.
+
+        """
+        machine = petname.generate()
+        prefix = ['multipass', 'exec', machine, '--']
+
+        check('multipass', 'launch', '--cpus', '2', '--mem', '8G',
+              self.DISTRO, '--name', machine)
+
+        check('multipass', 'copy-files', self.SNAP, '{}:'.format(machine))
+        check(*prefix, 'sudo', 'snap', 'install', '--classic',
+              '--{}'.format(channel), self.SNAP)
+
+        return machine, prefix
+
+    def test_cluster(self):
+
+        # After the setUp step, we should have a control node running
+        # in a multipass vm. Let's look up its cluster password and ip
+        # address.
+
+        openstack = '/snap/bin/microstack.openstack'
+
+        cluster_password = check_output(*self.PREFIX, 'sudo', 'snap',
+                                        'get', 'microstack',
+                                        'config.cluster.password')
+        control_ip = check_output(*self.PREFIX, 'sudo', 'snap',
+                                  'get', 'microstack',
+                                  'config.network.control-ip')
+
+        self.assertTrue(cluster_password)
+        self.assertTrue(control_ip)
+
+        compute_machine, compute_prefix = self._compute_node()
+
+        # TODO add the following to args for init
+        check(*compute_prefix, 'sudo', 'snap', 'set', 'microstack',
+              'config.network.control-ip={}'.format(control_ip))
+
+        check(*compute_prefix, 'sudo', 'microstack.init', '--compute',
+              '--cluster-password', cluster_password, '--debug')
+
+        # Verify that our services look setup properly on compute node.
+        services = check_output(
+            *compute_prefix, 'systemctl', 'status', 'snap.microstack.*',
+            '--no-page')
+
+        self.assertTrue('nova-compute' in services)
+        self.assertFalse('keystone-' in services)
+
+        check(*compute_prefix, '/snap/bin/microstack.launch', 'cirros',
+              '--name', 'breakfast', '--retry',
+              '--availability-zone', 'nova:{}'.format(compute_machine))
+
+        # TODO: verify horizon dashboard on control node.
+
+        # Verify endpoints
+        compute_ip = check_output(*compute_prefix, 'sudo', 'snap',
+                                  'get', 'microstack',
+                                  'config.network.compute-ip')
+        self.assertFalse(compute_ip == control_ip)
+
+        # Ping the instance
+        ip = None
+        servers = check_output(*compute_prefix, openstack,
+                               'server', 'list', '--format', 'json')
+        servers = json.loads(servers)
+        for server in servers:
+            if server['Name'] == 'breakfast':
+                ip = server['Networks'].split(",")[1].strip()
+                break
+
+        self.assertTrue(ip)
+
+        pings = 1
+        max_pings = 60  # ~1 minutes
+        # Ping the machine from the control node (we don't have
+        # networking wired up for the other nodes).
+        while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
+            pings += 1
+            if pings > max_pings:
+                self.assertFalse(
+                    True,
+                    msg='Max pings reached for instance on {}!'.format(
+                        compute_machine))
+
+        self.passed = True
+
+        # Compute machine cleanup
+        check('sudo', 'multipass', 'delete', compute_machine)
+
+
+if __name__ == '__main__':
+    # Run our tests, ignoring deprecation warnings and warnings about
+    # unclosed sockets. (TODO: setup a selenium server so that we can
+    # move from PhantomJS, which is deprecated, to to Selenium headless.)
+    unittest.main(warnings='ignore')

tests/test_control.py

@@ -26,6 +26,10 @@ class TestControlNode(Framework):
     INIT_FLAG = 'control'
 
     def test_control_node(self):
+        """A control node has all services running, so this shouldn't be any
+        different than our standard setup.
+
+        """
 
         print("Checking output of services ...")
         services = check_output(
@@ -35,7 +39,6 @@ class TestControlNode(Framework):
         print("services: @@@")
         print(services)
 
-        self.assertFalse('nova-' in services)
         self.assertTrue('neutron-' in services)
         self.assertTrue('keystone-' in services)
 

tools/cluster/cluster/client.py

@@ -0,0 +1,41 @@
+import json
+
+import requests
+
+from cluster.shell import check, check_output, write_tunnel_config
+
+
+def join():
+    """Join an existing cluster as a compute node."""
+    config = json.loads(check_output('snapctl', 'get', 'config'))
+
+    password = config['cluster']['password']
+    control_ip = config['network']['control-ip']
+    my_ip = config['network']['compute-ip']
+
+    if not password:
+        raise Exception("No cluster password specified!")
+
+    resp = requests.post(
+        'http://{}:10002/join'.format(control_ip),
+        json={'password': password, 'ip_address': my_ip})
+    if resp.status_code != 200:
+        # TODO better error and formatting.
+        raise Exception('Failed to get info from control node: {}'.format(
+            resp.json))
+    resp = resp.json()
+
+    # TODO: add better error handling to the below
+    os_password = resp['config']['credentials']['os-password']
+
+    # Write out tunnel config and restart neutron openvswitch agent.
+    write_tunnel_config(my_ip)
+    check('snapctl', 'restart', 'microstack.neutron-openvswitch-agent')
+
+    # Set passwords and such
+    check('snapctl', 'set', 'config.credentials.os-password={}'.format(
+        os_password))
+
+
+if __name__ == '__main__':
+    join()

tools/cluster/cluster/daemon.py

@@ -0,0 +1,55 @@
+import json
+
+from flask import Flask, request
+
+from cluster.shell import check, check_output, write_tunnel_config
+
+
+app = Flask(__name__)
+
+
+class Unauthorized(Exception):
+    pass
+
+
+def join_info(password, ip_address):
+    our_password = check_output('snapctl', 'get', 'config.cluster.password')
+
+    if password.strip() != our_password.strip():
+        raise Unauthorized()
+
+    # Load config
+    # TODO: be selective about what we return. For now, we just get everything.
+    config = json.loads(check_output('snapctl', 'get', 'config'))
+
+    # Write out tunnel config and restart neutron openvswitch agent.
+    write_tunnel_config(config['network']['control-ip'])
+    check('snapctl', 'restart', 'microstack.neutron-openvswitch-agent')
+
+    info = {'config': config}
+    return info
+
+
+@app.route('/')
+def home():
+    status = {
+        'status': 'running',
+        'info': 'Microstack clustering daemon.'
+
+    }
+    return json.dumps(status)
+
+
+@app.route('/join', methods=['POST'])
+def join():
+    req = request.json  # TODO: better error messages on failed parse.
+
+    password = req.get('password')
+    ip_address = req.get('ip_address')
+    if not password:
+        return 'No password specified', 500
+
+    try:
+        return json.dumps(join_info(password, ip_address))
+    except Unauthorized:
+        return (json.dumps({'error': 'Incorrect password.'}), 500)

tools/cluster/cluster/shell.py

@@ -0,0 +1,50 @@
+import os
+import pymysql
+import subprocess
+
+
+def sql(cmd) -> None:
+    """Execute some SQL!
+
+    Really simply wrapper around a pymysql connection, suitable for
+    passing the limited CREATE and GRANT commands that we need to pass
+    here.
+
+    :param cmd: sql to execute.
+
+    # TODO: move this into a shared shell library.
+
+    """
+    mysql_conf = '${SNAP_USER_COMMON}/etc/mysql/my.cnf'.format(**os.environ)
+    connection = pymysql.connect(host='localhost', user='root',
+                                 read_default_file=mysql_conf)
+
+    with connection.cursor() as cursor:
+        cursor.execute(cmd)
+
+
+def check_output(*args):
+    """Execute a shell command, returning the output of the command."""
+    return subprocess.check_output(args, env=os.environ,
+                                   universal_newlines=True).strip()
+
+
+def check(*args):
+    """Execute a shell command, raising an error on failed excution.
+
+    :param args: strings to be composed into the bash call.
+
+    """
+    return subprocess.check_call(args, env=os.environ)
+
+
+def write_tunnel_config(local_ip):
+    """Write tunnel config file for neutron agent."""
+
+    path_ = '{SNAP_COMMON}/etc/neutron/neutron.conf.d/tunnel.conf'.format(
+        **os.environ)
+    with open(path_, 'w') as file_:
+        file_.write("""\
+[OVS]
+local_ip = {local_ip}
+""".format(local_ip=local_ip))

tools/cluster/requirements.txt

@@ -0,0 +1,2 @@
+flask
+requests

tools/cluster/setup.py

@@ -0,0 +1,13 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="microstack_cluster",
+    description="Clustering client and server.",
+    packages=find_packages(exclude=("tests",)),
+    version="0.0.1",
+    entry_points={
+        'console_scripts': [
+            'microstack_join = cluster.client:join',
+        ],
+    }
+)

tools/init/init/main.py

@@ -28,23 +28,72 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
+
+import argparse
+import logging
+import secrets
+import string
 import sys
 
 from init.config import log
+from init.shell import check
 
 from init import questions
-from init.shell import check
 
-# Figure out whether to prompt for user input, and which type of node
-# we're running.
-# TODO drop in argparse and formalize this.
-COMPUTE = '--compute' in sys.argv
-CONTROL = '--control' in sys.argv
-AUTO = ('--auto' in sys.argv) or COMPUTE or CONTROL
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--auto', '-a', action='store_true',
+                        help='Run non interactively.')
+    parser.add_argument('--cluster-password')
+    parser.add_argument('--compute', action='store_true')
+    parser.add_argument('--control', action='store_true')
+    parser.add_argument('--debug', action='store_true')
+    args = parser.parse_args()
+    return args
+
+
+def process_args(args):
+    """Look through our args object and set the proper default config
+    values in our snap config, based on those args.
+
+    """
+    auto = args.auto or args.control or args.compute
+
+    if args.compute or args.control:
+        check('snapctl', 'set', 'config.clustered=true')
+
+    if args.compute:
+        check('snapctl', 'set', 'config.cluster.role=compute')
+
+    if args.control:
+        # If both compute and control are passed for some reason, we
+        # wind up with the role of 'control', which is best, as a
+        # control node also serves as a compute node in our hyper
+        # converged architecture.
+        check('snapctl', 'set', 'config.cluster.role=control')
+
+    if args.cluster_password:
+        check('snapctl', 'set', 'config.cluster.password={}'.format(
+            args.cluster_password))
+
+    if auto and not args.cluster_password:
+        alphabet = string.ascii_letters + string.digits
+        password = ''.join(secrets.choice(alphabet) for i in range(10))
+        check('snapctl', 'set', 'config.cluster.password={}'.format(password))
+
+    if args.debug:
+        log.setLevel(logging.DEBUG)
+
+    return auto
 
 
 def main() -> None:
+    args = parse_args()
+    auto = process_args(args)
+
     question_list = [
+        questions.Clustering(),
         questions.Dns(),
         questions.ExtGateway(),
         questions.ExtCidr(),
@@ -56,36 +105,19 @@ def main() -> None:
         # questions.FileHandleLimits(),
         questions.RabbitMq(),
         questions.DatabaseSetup(),
-        questions.NovaSetup(),
-        questions.NeutronSetup(),
+        questions.NovaHypervisor(),
+        questions.NovaControlPlane(),
+        questions.NeutronControlPlane(),
         questions.GlanceSetup(),
         questions.KeyPair(),
         questions.SecurityRules(),
         questions.PostSetup(),
     ]
 
-    # If we are setting up a "control" or "compute" node, override
-    # some of the default yes/no questions.
-    # TODO: move this into a nice little yaml parsing lib, and
-    # allow people to pass in a config file from the command line.
-    if CONTROL:
-        check('snapctl', 'set', 'questions.nova-setup=false')
-        check('snapctl', 'set', 'questions.key-pair=nil')
-        check('snapctl', 'set', 'questions.security-rules=false')
-
-    if COMPUTE:
-        check('snapctl', 'set', 'questions.rabbit-mq=false')
-        check('snapctl', 'set', 'questions.database-setup=false')
-        check('snapctl', 'set', 'questions.neutron-setup=false')
-        check('snapctl', 'set', 'questions.glance-setup=false')
-
     for question in question_list:
-        if AUTO:
-            # If we are automatically answering questions, replace the
-            # prompt for user input with a function that returns None,
-            # causing the question to fall back to the already set
-            # default
-            question._input_func = lambda prompt: None
+        if auto:
+            # Force all questions to be non-interactive if we passed --auto.
+            question.interactive = False
 
         try:
             question.ask()

tools/init/init/questions/clustering.py

@@ -0,0 +1,87 @@
+from getpass import getpass
+
+from init.questions.question import Question, InvalidAnswer
+from init.shell import check, check_output, fetch_ip_address
+
+
+class Role(Question):
+    _type = 'string'
+    config_key = 'config.cluster.role'
+    _question = "What is this machines' role? (control/compute)"
+    _valid_roles = ('control', 'compute')
+    interactive = True
+
+    def _input_func(self, prompt):
+        if not self.interactive:
+            return
+
+        for _ in range(0, 3):
+            role = input("{} > ".format(self._question))
+            if role in self._valid_roles:
+                return role
+
+            print('Role must be either "control" or "compute"')
+
+        raise InvalidAnswer('Too many failed attempts.')
+
+
+class Password(Question):
+    _type = 'string'  # TODO: type password support
+    config_key = 'config.cluster.password'
+    _question = 'Please enter a cluster password > '
+    interactive = True
+
+    def _input_func(self, prompt):
+        if not self.interactive:
+            return
+
+        # Get rid of 'default=' string the parent class has added to prompt.
+        prompt = self._question
+
+        for _ in range(0, 3):
+            password0 = getpass(prompt)
+            password1 = getpass('Please re-enter password > ')
+            if password0 == password1:
+                return password0
+
+            print("Passwords don't match!")
+
+        raise InvalidAnswer('Too many failed attempts.')
+
+
+class ControlIp(Question):
+    _type = 'string'
+    config_key = 'config.network.control-ip'
+    _question = 'Please enter the ip address of the control node'
+    interactive = True
+
+    def _load(self):
+        if check_output(
+                'snapctl', 'get', 'config.cluster.role') == 'control':
+            return fetch_ip_address() or super()._load()
+
+        return super()._load()
+
+
+class ComputeIp(Question):
+    _type = 'string'
+    config_key = 'config.network.compute-ip'
+    _question = 'Please enter the ip address of this node'
+    interactive = True
+
+    def _load(self):
+        if check_output(
+                'snapctl', 'get', 'config.cluster.role') == 'compute':
+            return fetch_ip_address() or super().load()
+
+        return super()._load()
+
+    def ask(self):
+        # If we are a control node, skip this question.
+        role = check_output('snapctl', 'get', Role.config_key)
+        if role == 'control':
+            ip = check_output('snapctl', 'get', ControlIp.config_key)
+            check('snapctl', 'set', '{}={}'.format(self.config_key, ip))
+            return
+
+        return super().ask()

tools/init/init/set_network_info.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env/python3
+from init.shell import default_network, check
+
+from init.config import log  # TODO name log.
+
+
+def main():
+    try:
+        ip, gate, cidr = default_network()
+    except Exception:
+        # TODO: more specific exception handling.
+        log.exception(
+            'Could not determine default network info. '
+            'Falling back on 10.20.20.1')
+        return
+
+    check('snapctl', 'set', 'config.network.ext-gateway={}'.format(gate))
+    check('snapctl', 'set', 'config.network.ext-cidr={}'.format(cidr))
+    check('snapctl', 'set', 'config.network.control-ip={}'.format(ip))
+    check('snapctl', 'set', 'config.network.control-ip={}'.format(ip))
+
+
+if __name__ == '__main__':
+    main()

tools/init/init/shell.py

@@ -27,6 +27,8 @@ import subprocess
 from time import sleep
 from typing import Dict, List
 
+import netaddr
+import netifaces
 import pymysql
 import wget
 
@@ -163,6 +165,42 @@ def restart(service: str) -> None:
     check('systemctl', 'restart', 'snap.microstack.{}'.format(service))
 
 
+def disable(service: str) -> None:
+    """Disable and mask a service.
+
+    :param service: the service(s) to be disabled. Can contain wild cards.
+                    e.g. *rabbit*
+
+    """
+    check('systemctl', 'disable', 'snap.microstack.{}'.format(service))
+    check('systemctl', 'mask', 'snap.microstack.{}'.format(service))
+
+
 def download(url: str, output: str) -> None:
     """Download a file to a path"""
     wget.download(url, output)
+
+
+def fetch_ip_address():
+    try:
+        interface = netifaces.gateways()['default'][netifaces.AF_INET][1]
+        return netifaces.ifaddresses(interface)[netifaces.AF_INET][0]['addr']
+    except (KeyError, IndexError):
+        log.exception('Failed to get ip address!')
+        return None
+
+
+def default_network():
+    """Get info about the default netowrk.
+
+    """
+    gateway, interface = netifaces.gateways()['default'][netifaces.AF_INET]
+    netmask = netifaces.ifaddresses(interface)[netifaces.AF_INET][0]['netmask']
+    ip_address = netifaces.ifaddresses(interface)[netifaces.AF_INET][0]['addr']
+    bits = netaddr.IPAddress(netmask).netmask_bits()
+    # TODO: better way to do this!
+    cidr = gateway.split('.')
+    cidr[-1] = '0/{}'.format(bits)
+    cidr = '.'.join(cidr)
+
+    return ip_address, gateway, cidr

tools/init/requirements.txt

@@ -1,3 +1,4 @@
+netaddr
+netifaces
 pymysql
 wget
-inflection

tools/init/setup.py

@@ -8,6 +8,7 @@ setup(
     entry_points={
         'console_scripts': [
             'microstack_init = init.main:main',
+            'set_network_info = init.set_network_info:main',
         ],
     },
 )

tools/init/tests/test_question.py

@@ -7,8 +7,8 @@ import mock
 # TODO: drop in test runner and get rid of this line.
 sys.path.append(os.getcwd())  # noqa
 
-from init.question import (Question, InvalidQuestion, InvalidAnswer,
-                           AnswerNotImplemented)
+from init.questions.question import (Question, InvalidQuestion, InvalidAnswer,
+                                     AnswerNotImplemented)
 
 
 ##############################################################################
@@ -20,10 +20,12 @@ from init.question import (Question, InvalidQuestion, InvalidAnswer,
 
 class InvalidTypeQuestion(Question):
     _type = 'foo'
+    config_key = 'invalid-type'
 
 
 class GoodAutoQuestion(Question):
     _type = 'auto'
+    config_key = 'good-auto-question'
 
     def yes(self, answer):
         return 'I am a good question!'
@@ -31,6 +33,7 @@ class GoodAutoQuestion(Question):
 
 class GoodBooleanQuestion(Question):
     _type = 'boolean'
+    config_key = 'good-bool-question'
 
     def yes(self, answer):
         return True
@@ -48,6 +51,7 @@ class GoodStringQuestion(Question):
 
     """
     _type = 'string'
+    config_key = 'good-string-question'
 
     def yes(self, answer):
         return answer
@@ -77,8 +81,8 @@ class TestQuestionClass(unittest.TestCase):
 
         self.assertTrue(GoodBooleanQuestion())
 
-    @mock.patch('init.question.shell.check_output')
-    @mock.patch('init.question.shell.check')
+    @mock.patch('init.questions.question.shell.check_output')
+    @mock.patch('init.questions.question.shell.check')
     def test_auto_question(self, mock_check, mock_check_output):
         mock_check_output.return_value = ''
 
@@ -93,8 +97,8 @@ class TestInput(unittest.TestCase):
     class's input handler.
 
     """
-    @mock.patch('init.question.shell.check_output')
-    @mock.patch('init.question.shell.check')
+    @mock.patch('init.questions.question.shell.check_output')
+    @mock.patch('init.questions.question.shell.check')
     def test_boolean_question(self, mock_check, mock_check_output):
         mock_check_output.return_value = 'true'
 
@@ -112,8 +116,8 @@ class TestInput(unittest.TestCase):
             q._input_func = lambda x: 'foo'
             q.ask()
 
-    @mock.patch('init.question.shell.check_output')
-    @mock.patch('init.question.shell.check')
+    @mock.patch('init.questions.question.shell.check_output')
+    @mock.patch('init.questions.question.shell.check')
     def test_string_question(self, mock_check, mock_check_output):
         mock_check_output.return_value = 'somedefault'
 

tools/launch/launch/main.py

@@ -43,6 +43,12 @@ def parse_args():
                         help='Wait for server to become active before exiting')
     parser.add_argument('-r', '--retry', action='store_true',
                         help='Retry failed launch attempts')
+    # TODO: add a passthrough for other openstack 'server create'
+    # args. Manually specifying them here is a bit silly.  For now, we
+    # need to specify availability zone in some tests, so we add it
+    # here.
+    parser.add_argument('--availability-zone',
+                        help='passthrough to avail zone')
 
     args = parser.parse_args()
     return args
@@ -50,13 +56,18 @@ def parse_args():
 
 def create_server(name, args):
 
-    ret = check_output('openstack', 'server', 'create',
-                       '--flavor', args.flavor,
-                       '--image', args.image,
-                       '--nic', 'net-id={}'.format(args.net_id),
-                       '--key-name', args.key,
-                       name, '--format', 'json')
-    ret = json.loads(ret)
+    cmd = [
+        'openstack', 'server', 'create',
+        '--flavor', args.flavor,
+        '--image', args.image,
+        '--nic', 'net-id={}'.format(args.net_id),
+        '--key-name', args.key,
+        name, '--format', 'json'
+    ]
+    if args.availability_zone:
+        cmd += ['--availability-zone', args.availability_zone]
+
+    ret = json.loads(check_output(*cmd))
     return ret['id']
 
 
@@ -141,7 +152,7 @@ Server {} launched! (status is {})
 Access it with `ssh -i \
 $HOME/.ssh/id_microstack` <username>@{}""".format(name, status, ip))
 
-    gate = check_output('snapctl', 'get', 'questions.ext-gateway')
+    gate = check_output('snapctl', 'get', 'config.network.ext-gateway')
     print('You can also visit the OpenStack dashboard at http://{}'.format(
         gate))
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = init_lint, init_unit, multipass
+envlist = lint, unit, multipass
 skipsdist = True
 
 [testenv]
@@ -24,7 +24,6 @@ commands =
     # Specify tests in sequence, as they can't run in parallel if not
     # using multipass.
     {toxinidir}/tests/test_basic.py
-    {toxinidir}/tests/test_control.py
 
 [testenv:multipass]
 # Default testing environment for a human operated machine. Builds the
@@ -42,7 +41,6 @@ commands =
     {toxinidir}/tools/multipass_build.sh
     flake8 {toxinidir}/tests/
     {toxinidir}/tests/test_basic.py
-    {toxinidir}/tests/test_control.py
 
 [testenv:basic]
 # Just run basic_test.sh, with multipass support.
@@ -54,7 +52,30 @@ commands =
     {toxinidir}/tools/basic_setup.sh
     flake8 {toxinidir}/tests/
     {toxinidir}/tests/test_basic.py
-    {toxinidir}/tests/test_control.py
+
+[testenv:cluster]
+# Test out clustering!
+# Requires multipass.
+deps = -r{toxinidir}/test-requirements.txt
+setenv =
+    MULTIPASS=true
+
+commands =
+    {toxinidir}/tools/basic_setup.sh
+    flake8 {toxinidir}/tests/
+    {toxinidir}/tests/test_cluster.py
+
+[testenv:build]
+# Just run basic_test.sh, with multipass support.
+deps = -r{toxinidir}/test-requirements.txt
+setenv =
+    MULTIPASS=true
+
+commands =
+    flake8 {toxinidir}/tests/
+    flake8 {toxinidir}/tools/init/init
+    flake8 {toxinidir}/tools/cluster/cluster
+    {toxinidir}/tools/multipass_build.sh
 
 [testenv:lint]
 deps = -r{toxinidir}/test-requirements.txt
@@ -62,13 +83,9 @@ commands =
     flake8 {toxinidir}/tests/
     flake8 {toxinidir}/tools/init/init/
     flake8 {toxinidir}/tools/launch/launch/
+    flake8 {toxinidir}/tools/cluster/cluster/
 
-[testenv:init_lint]
-deps = -r{toxinidir}/tools/init/test-requirements.txt
-       -r{toxinidir}/tools/init/requirements.txt
-commands = flake8 {toxinidir}/tools/init/init/
-
-[testenv:init_unit]
+[testenv:unit]
 deps = -r{toxinidir}/tools/init/test-requirements.txt
        -r{toxinidir}/tools/init/requirements.txt
 commands =