Enable HA with Manila

An indirect requirement of enabling HA with Manila
is migrating from directly using WSGI to using
mod-wsgi with Apache.

Change-Id: I1f501283db3db1338d47a89a7688cf5035d08a7a
Closes-Bug: #1867358

src/lib/charm/openstack/manila.py

@@ -18,9 +18,11 @@
 from __future__ import absolute_import
 
 import collections
+import os
 import re
 import subprocess
 
+import charmhelpers.core.host as host
 import charms_openstack.charm
 import charms_openstack.adapters
 import charms_openstack.ip as os_ip
@@ -32,12 +34,16 @@ PACKAGES = ['manila-api',
             'manila-share',
             'python-pymysql',
             'python-apt',  # for subordinate neutron-openvswitch if needed.
+            'apache2',
+            'libapache2-mod-wsgi',
             ]
 
 MANILA_DIR = '/etc/manila/'
 MANILA_CONF = MANILA_DIR + "manila.conf"
 MANILA_LOGGING_CONF = MANILA_DIR + "logging.conf"
 MANILA_API_PASTE_CONF = MANILA_DIR + "api-paste.ini"
+MANILA_WEBSERVER_SITE = 'manila-api'
+MANILA_WSGI_CONF = '/etc/apache2/sites-available/manila-api.conf'
 
 # select the default release function and ssl feature
 charms_openstack.charm.use_defaults('charm.default-select-release')
@@ -213,7 +219,8 @@ class ManilaCharm(charms_openstack.charm.HAOpenStackCharm):
 
     @property
     def services(self):
-        services = ['manila-api',
+        services = ['apache2',
+                    'haproxy',
                     'manila-scheduler',
                     'manila-data']
         if not self.get_adapter('remote-manila-plugin.available'):
@@ -225,10 +232,12 @@ class ManilaCharm(charms_openstack.charm.HAOpenStackCharm):
         services = self.services
         return {
             MANILA_CONF: services,
-            MANILA_API_PASTE_CONF: services,
+            MANILA_API_PASTE_CONF: ['apache2'],
             MANILA_LOGGING_CONF: services,
+            MANILA_WSGI_CONF: ['apache2'],
         }
-    # ha_resources = ['vips', 'haproxy']
+
+    ha_resources = ['vips', 'haproxy', 'dnsha']
 
     # Custom charm configuration
 
@@ -242,6 +251,7 @@ class ManilaCharm(charms_openstack.charm.HAOpenStackCharm):
         # this creates the /etc/nova directory for the
         # neutron-openvswitch plugin if needed.
         subprocess.check_call(["mkdir", "-p", "/etc/nova"])
+        host.service_pause('manila-api')
         self.assess_status()
 
     def custom_assess_status_check(self):
@@ -445,6 +455,18 @@ class ManilaCharm(charms_openstack.charm.HAOpenStackCharm):
         return self.get_adapter('manila-plugin.available') or \
             self.get_adapter('remote-manila-plugin.available')
 
+    def enable_webserver_site(self):
+        """Enable Manila API apache2 site if rendered or installed"""
+        if os.path.exists(MANILA_WSGI_CONF):
+            check_enabled = subprocess.call(
+                ['a2query', '-s', MANILA_WEBSERVER_SITE]
+            )
+            if check_enabled != 0:
+                subprocess.check_call(['a2ensite',
+                                       MANILA_WEBSERVER_SITE])
+                host.service_reload('apache2',
+                                    restart_on_failure=True)
+
 
 class ManilaCharmRocky(ManilaCharm):
 
@@ -456,12 +478,15 @@ class ManilaCharmRocky(ManilaCharm):
         'manila-scheduler',
         'manila-share',
         'python3-manila',
+        'apache2',
+        'libapache2-mod-wsgi-py3',
     ]
 
     purge_packages = [
         'python-manila',
         'python-memcache',
         'python-pymysql',
+        'libapache2-mod-wsgi',
     ]
 
     python_version = 3

src/reactive/manila_handlers.py

@@ -20,21 +20,25 @@ import charmhelpers.core.host as ch_host
 
 import charms.reactive
 import charms.reactive.relations as relations
+import charms_openstack.bus
 import charms_openstack.charm
 
-# This charm's library contains all of the handler code associated with
-# manila -- we need to import it to get the definitions for the charm.
-import charm.openstack.manila  # noqa
+charms_openstack.bus.discover()
 
 
 # Use the charms.openstack defaults for common states and hooks
 charms_openstack.charm.use_defaults(
-    'charm.installed',
     'amqp.connected',
+    'certificates.available',
+    'charm.installed',
+    'cluster.available',
+    'config.rendered',
     'shared-db.connected',
     'upgrade-charm',
     'certificates.available',
     'cluster.available',
+    'config.changed',
+    'update-status',
 )
 
 
@@ -96,6 +100,7 @@ def maybe_do_syncdb(shared_db):
     """
     with charms_openstack.charm.provide_charm_instance() as manila_charm:
         manila_charm.db_sync()
+    charms.reactive.set_state('db.synced')
 
 
 @charms.reactive.when('shared-db.available',
@@ -118,6 +123,7 @@ def render_stuff(*args):
             relations.endpoint_from_flag('remote-manila-plugin.changed')
         if manila_plugin:
             manila_plugin.clear_changed()
+        manila_charm.enable_webserver_site()
 
 
 @charms.reactive.when('shared-db.available',
@@ -160,3 +166,27 @@ def update_status():
         if state == 'blocked' and services:
             # try to start the 'manila-share' service
             ch_host.service_start('manila-share')
+
+
+@charms.reactive.when('db.synced', 'manila.config.rendered')
+@charms.reactive.when_not('config.rendered')
+def config_rendered():
+    """Set the config.rendered state when ready for operation.
+
+    The config.rendered flag is used by the default handlers in
+    charms.openstack to enable / disable services based on the
+    readiness of the deployment. The Manila charm is using this
+    functionalty to ensure that the Manila services start up only
+    after the database has been synced to remove a race condition
+    where the services won't restart after failing several times
+    with missing migrations.
+    """
+    charms.reactive.set_state('config.rendered')
+
+
+@charms.reactive.when('ha.connected')
+def cluster_connected(hacluster):
+    """Configure HA resources in corosync"""
+    with charms_openstack.charm.provide_charm_instance() as manila_charm:
+        manila_charm.configure_ha_resources(hacluster)
+        manila_charm.assess_status()

src/templates/pike/manila-api.conf

@@ -0,0 +1,19 @@
+Listen {{ options.service_listen_info.manila_api.public_port }}
+
+# workaround problem with Python Cryptography and libssl1.0.0 by adding
+# WSGIApplicationGroup %{GLOBAL}
+# See https://cryptography.io/en/latest/faq/#starting-cryptography-using-mod-wsgi-produces-an-internalerror-during-a-call-in-register-osrandom-engine
+
+<VirtualHost *:{{ options.service_listen_info.manila_api.public_port }}>
+    WSGIDaemonProcess manila-api processes={{ options.wsgi_worker_context.processes }} threads=1 user=manila group=manila display-name=%{GROUP}
+    WSGIProcessGroup manila-api
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIScriptAlias / /usr/bin/manila-wsgi
+    <Directory /usr/bin>
+        <Files manila-wsgi>
+            Require all granted
+        </Files>
+    </Directory>
+    ErrorLog /var/log/apache2/manila_error.log
+    CustomLog /var/log/apache2/manila_access.log combined
+</VirtualHost>

src/tests/bundles/bionic-queens.yaml

@@ -2,7 +2,7 @@ series: bionic
 services:
   manila:
     charm: ../../../manila
-    num_units: 1
+    num_units: 3
     series: bionic
     options:
       default-share-backend: generic

src/tests/bundles/bionic-rocky.yaml

@@ -26,7 +26,7 @@ services:
       source: *source
   manila:
     charm: ../../../manila
-    num_units: 1
+    num_units: 3
     series: bionic
     options:
       openstack-origin: *source
@@ -83,7 +83,6 @@ services:
     num_units: 1
     options:
       openstack-origin: *source
-
 relations:
   - - ceph-mon
     - ceph-osd
@@ -144,4 +143,4 @@ relations:
   - - 'nova-cloud-controller:image-service'
     - 'glance:image-service'
   - - 'nova-cloud-controller:quantum-network-service'
-    - 'neutron-gateway:quantum-network-service'
+    - 'neutron-gateway:quantum-network-service'

src/tests/bundles/bionic-stein.yaml

@@ -26,7 +26,7 @@ services:
       source: *source
   manila:
     charm: ../../../manila
-    num_units: 1
+    num_units: 3
     series: bionic
     options:
       openstack-origin: *source

src/tests/bundles/bionic-train.yaml

@@ -26,7 +26,7 @@ services:
       source: *source
   manila:
     charm: ../../../manila
-    num_units: 1
+    num_units: 3
     series: bionic
     options:
       openstack-origin: *source
@@ -85,7 +85,7 @@ services:
       openstack-origin: *source
   placement:
     charm: cs:~openstack-charmers-next/placement
-    num_units: 1
+    num_units: 3
     options:
       openstack-origin: *source
 relations:

src/tests/bundles/overlays/local-charm-overlay.yaml.j2

@@ -0,0 +1,11 @@
+# Add True HA
+applications:
+  manila:
+    options:
+      vip: '{{ OS_VIP00 }}'
+  hacluster:
+    charm: cs:~openstack-charmers-next/hacluster
+    num_units: 0
+relations:
+- - manila
+  - hacluster

src/tests/bundles/xenial-mitaka.yaml

@@ -4,7 +4,7 @@ variables:
 services:
   manila:
     charm: ../../../manila
-    num_units: 1
+    num_units: 3
     series: *series
     options:
       default-share-backend: generic
@@ -39,4 +39,4 @@ relations:
   - - keystone
     - percona-cluster
   - - manila
-    - percona-cluster
+    - percona-cluster

src/tests/bundles/xenial-ocata.yaml

@@ -4,7 +4,7 @@ variables:
 services:
   manila:
     charm: ../../../manila
-    num_units: 1
+    num_units: 3
     series: *series
     options:
       default-share-backend: generic

src/tests/bundles/xenial-pike.yaml

@@ -4,7 +4,7 @@ variables:
 services:
   manila:
     charm: ../../../manila
-    num_units: 1
+    num_units: 3
     series: *series
     options:
       default-share-backend: generic
@@ -28,7 +28,6 @@ services:
     num_units: 1
     options:
       openstack-origin: *source
-
 relations:
   - - manila:manila-plugin
     - manila-generic

src/tests/bundles/xenial-queens.yaml

@@ -4,7 +4,7 @@ variables:
 services:
   manila:
     charm: ../../../manila
-    num_units: 1
+    num_units: 3
     series: *series
     options:
       default-share-backend: generic

src/tests/tests.yaml

@@ -2,8 +2,9 @@ charm_name: manila
 dev_bundles:
 - bionic
 gate_bundles:
-- xenial-mitaka
-- xenial-ocata
+# Manila doesn't support mod-wsgi before Pike
+# - xenial-mitaka
+# - xenial-ocata
 - xenial-pike
 - xenial-queens
 - bionic-queens

unit_tests/test_manila_handlers.py

@@ -43,10 +43,14 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks):
                                     'manila.config.rendered', ),
                 'config_changed': ('shared-db.available',
                                    'identity-service.available',
-                                   'amqp.available', )
+                                   'amqp.available', ),
+                'config_rendered': ('db.synced', 'manila.config.rendered',),
+                'cluster_connected': ('ha.connected',)
             },
             'when_not': {
                 'register_endpoints': ('identity-service.available', ),
+                'maybe_do_syncdb': ('db.synced',),
+                'config_rendered': ('config.rendered',)
             },
             'when_any': {
                 'config_changed': ('config-changed',