Add support for HA deployments using etcd

Add support for relating vault to etcd to support HA deployments
of vault.

Depends-On: Iebb2415077b682dfdf590b4b5f35a3c593ed3d69
Depends-On: I05a04bdc237b2a698b2f2d29e18c5a33510a2513
Change-Id: I161db8296031776652425f563a11de3ee8f7f86e

src/layer.yaml

@@ -4,6 +4,7 @@ includes:
  - interface:nrpe-external-master
  - interface:pgsql
  - interface:mysql-shared
+ - interface:etcd
 options:
   basic:
     packages:

src/metadata.yaml

@@ -21,6 +21,8 @@ requires:
     interface: pgsql
   shared-db:
     interface: mysql-shared
+  etcd:
+    interface: etcd
 provides:
   nrpe-external-master:
     interface: nrpe-external-master

src/reactive/vault.py

@@ -1,7 +1,9 @@
 import base64
+import hvac
 import psycopg2
 import subprocess
 
+
 from charmhelpers.contrib.charmsupport.nrpe import (
     NRPE,
     add_init_service_checks,
@@ -10,12 +12,17 @@ from charmhelpers.contrib.charmsupport.nrpe import (
 )
 
 from charmhelpers.core.hookenv import (
+    DEBUG,
     config,
+    log,
     open_port,
     status_set,
+    unit_private_ip,
 )
 
 from charmhelpers.core.host import (
+    service_restart,
+    service_running,
     service_start,
     write_file,
 )
@@ -33,6 +40,10 @@ from charms.reactive import (
     when_not,
 )
 
+from charms.reactive.relations import (
+    endpoint_from_flag,
+)
+
 # See https://www.vaultproject.io/docs/configuration/storage/postgresql.html
 
 VAULT_TABLE_DDL = """
@@ -50,6 +61,24 @@ CREATE INDEX IF NOT EXISTS parent_path_idx ON vault_kv_store (parent_path);
 """
 
 
+def get_client():
+    return hvac.Client(url=get_api_url())
+
+
+def can_restart():
+    safe_restart = False
+    if not service_running('vault'):
+        safe_restart = True
+    else:
+        client = get_client()
+        if not client.is_initialized():
+            safe_restart = True
+        elif client.is_sealed():
+            safe_restart = True
+    log("Safe to restart: {}".format(safe_restart), level=DEBUG)
+    return safe_restart
+
+
 def ssl_available(config):
     if '' in (config['ssl-cert'], config['ssl-key']):
         return False
@@ -59,7 +88,27 @@ def ssl_available(config):
 def configure_vault(context):
     context['disable_mlock'] = config()['disable-mlock']
     context['ssl_available'] = is_state('vault.ssl.available')
+    log("Running configure_vault", level=DEBUG)
+    context['disable_mlock'] = config()['disable-mlock']
+    context['ssl_available'] = is_state('vault.ssl.available')
+    etcd = endpoint_from_flag('etcd.available')
+    if etcd:
+        log("Etcd detected, adding to context", level=DEBUG)
+        context['etcd_conn'] = etcd.connection_string()
+        context['etcd_tls_ca_file'] = '/var/snap/vault/common/etcd-ca.pem'
+        context['etcd_tls_cert_file'] = '/var/snap/vault/common/etcd-cert.pem'
+        context['etcd_tls_key_file'] = '/var/snap/vault/common/etcd.key'
+        etcd.save_client_credentials(
+            context['etcd_tls_key_file'],
+            context['etcd_tls_cert_file'],
+            context['etcd_tls_ca_file'])
+        context['vault_api_url'] = get_api_url()
+        log("Etcd detected, setting vault_api_url to {}".format(
+            context['vault_api_url']))
+    else:
+        log("Etcd not detected", level=DEBUG)
     status_set('maintenance', 'creating vault config')
+    log("Rendering vault.hcl.j2", level=DEBUG)
     render(
         'vault.hcl.j2',
         '/var/snap/vault/common/vault.hcl',
@@ -72,7 +121,10 @@ def configure_vault(context):
         {},
         perms=0o644)
     status_set('maintenance', 'starting vault')
-    service_start('vault')      # restart seals the vault
+    if can_restart():
+        service_restart('vault')
+    else:
+        service_start('vault')      # restart seals the vault
     status_set('maintenance', 'opening vault port')
     open_port(8200)
     set_state('configured')
@@ -84,6 +136,15 @@ def configure_vault(context):
         status_set('active', '=^_^=')
 
 
+def get_api_url():
+    protocol = 'http'
+    port = '8200'
+    ip = unit_private_ip()
+    if is_state('vault.ssl.available'):
+        protocol = 'https'
+    return '{}://{}:{}'.format(protocol, ip, port)
+
+
 @when('snap.installed.vault')
 @when_not('configured')
 @when('db.master.available')
@@ -207,6 +268,24 @@ def ssl_ca_changed():
     remove_state('vault.ssl.configured')
 
 
+@when_not('etcd.local.configured')
+@when('etcd.available')
+def etcd_setup(etcd):
+    log("Detected etcd.available, removing configured", level=DEBUG)
+    remove_state('configured')
+    remove_state('etcd.local.unconfigured')
+    set_state('etcd.local.configured')
+
+
+@when_not('etcd.local.unconfigured')
+@when_not('etcd.available')
+def etcd_not_ready():
+    log("Detected etcd_not_ready, removing configured", level=DEBUG)
+    set_state('etcd.local.unconfigured')
+    remove_state('etcd.local.configured')
+    remove_state('configured')
+
+
 @when('configured')
 @when('nrpe-external-master.available')
 @when_not('vault.nrpe.configured')

src/templates/vault.hcl.j2

@@ -1,3 +1,6 @@
+{%- if vault_api_url %}
+api_addr = "{{ vault_api_url }}"
+{%- endif %}
 {%- if disable_mlock %}
 disable_mlock = true
 {%- endif %}
@@ -13,6 +16,16 @@ storage "mysql" {
   address = "{{ mysql_db_relation.db_host() }}:3306"
 }
 {%- endif %}
+{%- if etcd_conn %}
+ha_storage "etcd" {
+  ha_enabled = "true"
+  address  = "{{ etcd_conn }}"
+  tls_ca_file  = "{{ etcd_tls_ca_file }}"
+  tls_cert_file  = "{{ etcd_tls_cert_file }}"
+  tls_key_file  = "{{ etcd_tls_key_file }}"
+  etcd_api = "v3"
+}
+{%- endif %}
 listener "tcp" {
  address = "0.0.0.0:8200"
 {%- if ssl_available %}

src/tests/bundles/xenial-ha-mysql.yaml

@@ -0,0 +1,24 @@
+series: xenial
+services:
+  vault:
+    num_units: 3
+    series: xenial
+    charm: ../../../vault
+  mysql:
+    charm: cs:mysql
+    num_units: 1
+  easyrsa:
+    charm: cs:~containers/easyrsa
+    num_units: 1
+  etcd:
+    charm: cs:etcd
+    num_units: 2
+    options:
+      channel: 3.1/stable
+relations:
+- - vault:shared-db
+  - mysql:shared-db
+- - etcd:certificates
+  - easyrsa:client
+- - etcd:db
+  - vault:etcd

src/tests/tests.yaml

@@ -4,6 +4,7 @@ tests:
 configure:
   - zaza.charm_tests.vault.setup.basic_setup
 gate_bundles:
+  - xenial-ha-mysql
   - xenial-postgres
   - xenial-mysql
 smoke_bundles:

src/wheelhouse.txt

@@ -0,0 +1 @@
+hvac

test-requirements.txt

@@ -1,4 +1,5 @@
 # Unit test requirements
+hvac
 flake8>=2.2.4,<=2.4.1
 os-testr>=0.4.1
 charms.reactive

unit_tests/test_vault.py

@@ -16,6 +16,35 @@ import reactive.vault as handlers  # noqa: E402
 
 class TestHandlers(unittest.TestCase):
 
+    def setUp(self):
+        super(TestHandlers, self).setUp()
+        self.patches = [
+            'config',
+            'endpoint_from_flag',
+            'is_state',
+            'log',
+            'open_port',
+            'service_restart',
+            'service_running',
+            'service_start',
+            'set_state',
+            'status_set',
+            'remove_state',
+            'render',
+            'unit_private_ip',
+        ]
+        self.patch_all()
+
+    def _patch(self, method):
+        _m = patch.object(handlers, method)
+        mock = _m.start()
+        self.addCleanup(_m.stop)
+        return mock
+
+    def patch_all(self):
+        for method in self.patches:
+            setattr(self, method, self._patch(method))
+
     def test_ssl_available(self):
         self.assertFalse(handlers.ssl_available({
             'ssl-cert': '',
@@ -30,20 +59,15 @@ class TestHandlers(unittest.TestCase):
             'ssl-cert': 'acert',
             'ssl-key': 'akey'}))
 
-    @patch.object(handlers, 'is_state')
-    @patch.object(handlers, 'config')
-    @patch.object(handlers, 'open_port')
-    @patch.object(handlers, 'service_start')
-    @patch.object(handlers, 'render')
-    @patch.object(handlers, 'status_set')
-    @patch.object(handlers, 'remove_state')
-    def test_configure_vault(self, remove_state, status_set, render,
-                             service_start, open_port, config, is_state):
-        config.return_value = {'disable-mlock': False}
-        is_state.return_value = True
+    @patch.object(handlers, 'can_restart')
+    def test_configure_vault(self, can_restart):
+        can_restart.return_value = True
+        self.config.return_value = {'disable-mlock': False}
+        self.is_state.return_value = True
         db_context = {
             'storage_name': 'psql',
             'psql_db_conn': 'myuri'}
+        self.endpoint_from_flag.return_value = None
         handlers.configure_vault(db_context)
         expected_context = {
             'storage_name': 'psql',
@@ -70,12 +94,12 @@ class TestHandlers(unittest.TestCase):
                 {},
                 perms=0o644)
         ]
-        open_port.assert_called_once_with(8200)
-        status_set.assert_has_calls(status_set_calls)
-        render.assert_has_calls(render_calls)
+        self.open_port.assert_called_once_with(8200)
+        self.status_set.assert_has_calls(status_set_calls)
+        self.render.assert_has_calls(render_calls)
 
         # Check flipping disable-mlock makes it to the context
-        config.return_value = {'disable-mlock': True}
+        self.config.return_value = {'disable-mlock': True}
         expected_context['disable_mlock'] = True
         handlers.configure_vault(db_context)
         render_calls = [
@@ -90,7 +114,7 @@ class TestHandlers(unittest.TestCase):
                 {},
                 perms=0o644)
         ]
-        render.assert_has_calls(render_calls)
+        self.render.assert_has_calls(render_calls)
 
     @patch.object(handlers, 'configure_vault')
     def test_configure_vault_psql(self, configure_vault):
@@ -109,28 +133,24 @@ class TestHandlers(unittest.TestCase):
             'storage_name': 'mysql',
             'mysql_db_relation': mysql})
 
-    @patch.object(handlers, 'remove_state')
-    def test_disable_mlock_changed(self, remove_state):
+    def test_disable_mlock_changed(self):
         handlers.disable_mlock_changed()
-        remove_state.assert_called_once_with('configured')
+        self.remove_state.assert_called_once_with('configured')
 
-    @patch.object(handlers, 'remove_state')
-    def test_upgrade_charm(self, remove_state):
+    def test_upgrade_charm(self):
         calls = [mock.call('configured'),
                  mock.call('vault.nrpe.configured'),
                  mock.call('vault.ssl.configured')]
         handlers.upgrade_charm()
-        remove_state.assert_has_calls(calls)
+        self.remove_state.assert_has_calls(calls)
 
     def test_request_db(self):
         psql = mock.MagicMock()
         handlers.request_db(psql)
         psql.set_database.assert_called_once_with('vault')
 
-    @patch.object(handlers, 'set_state')
     @patch.object(handlers, 'psycopg2')
-    @patch.object(handlers, 'status_set')
-    def test_create_vault_table(self, status_set, psycopg2, set_state):
+    def test_create_vault_table(self, psycopg2):
         psql = mock.MagicMock()
         psql.master = 'myuri'
         handlers.create_vault_table(psql)
@@ -140,7 +160,83 @@ class TestHandlers(unittest.TestCase):
         ]
         psycopg2.connect().cursor().execute.assert_has_calls(db_calls)
 
-    @patch.object(handlers, 'remove_state')
-    def test_database_not_ready(self, remove_state):
+    def test_database_not_ready(self):
         handlers.database_not_ready()
-        remove_state.assert_called_once_with('vault.schema.created')
+        self.remove_state.assert_called_once_with('vault.schema.created')
+
+    @patch.object(handlers, 'can_restart')
+    @patch.object(handlers, 'get_api_url')
+    def test_configure_vault_etcd(self, get_api_url, can_restart):
+        can_restart.return_value = True
+        get_api_url.return_value = 'http://this-unit'
+        self.config.return_value = {'disable-mlock': False}
+        etcd_mock = mock.MagicMock()
+        etcd_mock.connection_string.return_value = 'http://etcd'
+        self.endpoint_from_flag.return_value = etcd_mock
+        self.is_state.return_value = True
+        handlers.configure_vault({})
+        expected_context = {
+            'disable_mlock': False,
+            'ssl_available': True,
+            'etcd_conn': 'http://etcd',
+            'etcd_tls_ca_file': '/var/snap/vault/common/etcd-ca.pem',
+            'etcd_tls_cert_file': '/var/snap/vault/common/etcd-cert.pem',
+            'etcd_tls_key_file': '/var/snap/vault/common/etcd.key',
+            'vault_api_url': 'http://this-unit'}
+        render_calls = [
+            mock.call(
+                'vault.hcl.j2',
+                '/var/snap/vault/common/vault.hcl',
+                expected_context,
+                perms=0o600),
+            mock.call(
+                'vault.service.j2',
+                '/etc/systemd/system/vault.service',
+                {},
+                perms=0o644)
+        ]
+        self.render.assert_has_calls(render_calls)
+
+    @patch.object(handlers.hvac, 'Client')
+    @patch.object(handlers, 'get_api_url')
+    def test_get_client(self, get_api_url, hvac_Client):
+        get_api_url.return_value = 'http://this-unit'
+        handlers.get_client()
+        hvac_Client.assert_called_once_with(url='http://this-unit')
+
+    def test_can_restart_vault_down(self):
+        self.service_running.return_value = False
+        self.assertTrue(handlers.can_restart())
+
+    @patch.object(handlers, 'get_client')
+    def test_can_restart_not_initialized(self, get_client):
+        hvac_mock = mock.MagicMock()
+        hvac_mock.is_initialized.return_value = False
+        get_client.return_value = hvac_mock
+        self.assertTrue(handlers.can_restart())
+
+    @patch.object(handlers, 'get_client')
+    def test_can_restart_sealed(self, get_client):
+        hvac_mock = mock.MagicMock()
+        hvac_mock.is_initialized.return_value = True
+        hvac_mock.is_sealed.return_value = True
+        get_client.return_value = hvac_mock
+        self.assertTrue(handlers.can_restart())
+
+    @patch.object(handlers, 'get_client')
+    def test_can_restart_unsealed(self, get_client):
+        hvac_mock = mock.MagicMock()
+        hvac_mock.is_initialized.return_value = True
+        hvac_mock.is_sealed.return_value = False
+        get_client.return_value = hvac_mock
+        self.assertFalse(handlers.can_restart())
+
+    def test_get_api_url_ssl(self):
+        self.is_state.return_value = True
+        self.unit_private_ip.return_value = '1.2.3.4'
+        self.assertEqual(handlers.get_api_url(), 'https://1.2.3.4:8200')
+
+    def test_get_api_url_nossl(self):
+        self.is_state.return_value = False
+        self.unit_private_ip.return_value = '1.2.3.4'
+        self.assertEqual(handlers.get_api_url(), 'http://1.2.3.4:8200')