Add disable-aws-compat config to disable AWS compatibility layer

For OpenStack >= Liberty EC2 and objectstore services aren't installed and
deprecated, but there is no way to disable it, this patch adds a way to
permanently disable and enable them

Change-Id: Ie78ba5af1d953e8ebed9195202522ff73efcde9f
Closes-Bug: #1533255

config.yaml

@@ -421,3 +421,10 @@ options:
     description: |
       Apply system hardening. Supports a space-delimited list of modules
       to run. Supported modules currently include os, ssh, apache and mysql.
+  disable-aws-compat:
+    default: false
+    type: boolean
+    description: |
+      For OpenStack Icehouse, Juno and Kilo by default a compatibility layer
+      for EC2 and S3 is configured, setting this option to `true` the services are
+      stopped and disabled.

hooks/nova_cc_hooks.py

@@ -111,6 +111,7 @@ from nova_cc_utils import (
     setup_ipv6,
     is_db_initialised,
     assess_status,
+    update_aws_compat_services,
 )
 
 from charmhelpers.contrib.hahelpers.cluster import (
@@ -289,6 +290,7 @@ def config_changed():
                 compute_changed(rid, unit)
 
     update_nova_consoleauth_config()
+    update_aws_compat_services()
 
 
 @hooks.hook('amqp-relation-joined')

hooks/nova_cc_utils.py

@@ -63,7 +63,8 @@ from charmhelpers.fetch import (
     apt_upgrade,
     apt_update,
     apt_install,
-    add_source
+    add_source,
+    filter_installed_packages
 )
 
 from charmhelpers.core.hookenv import (
@@ -87,9 +88,11 @@ from charmhelpers.core.host import (
     add_user_to_group,
     mkdir,
     service,
+    service_pause,
+    service_resume,
+    service_running,
     service_start,
     service_stop,
-    service_running,
     lsb_release,
 )
 
@@ -182,8 +185,9 @@ BASE_SERVICES = [
     'nova-conductor',
 ]
 
+AWS_COMPAT_SERVICES = ['nova-api-ec2', 'nova-objectstore']
 SERVICE_BLACKLIST = {
-    'liberty': ['nova-api-ec2', 'nova-objectstore']
+    'liberty': AWS_COMPAT_SERVICES
 }
 
 API_PORTS = {
@@ -207,7 +211,7 @@ def resolve_services():
     _services = deepcopy(BASE_SERVICES)
     os_rel = os_release('nova-common')
     for release in SERVICE_BLACKLIST:
-        if os_rel >= release:
+        if os_rel >= release or config('disable-aws-compat'):
             [_services.remove(service)
              for service in SERVICE_BLACKLIST[release]]
     return _services
@@ -1346,3 +1350,23 @@ def _pause_resume_helper(f, configs):
     f(assess_status_func(configs),
       services=services(),
       ports=None)
+
+
+def update_aws_compat_services():
+    """Depending on the configuration of `disable-aws-compatibility` config
+    option.
+
+    This will stop/start and disable/enable `nova-api-ec2` and
+    `nova-objectstore` services.
+    """
+    # if packages aren't installed, then there is nothing to do
+    if filter_installed_packages(AWS_COMPAT_SERVICES) != []:
+        return
+
+    if config('disable-aws-compat'):
+        # TODO: the endpoints have to removed from keystone
+        for service_ in AWS_COMPAT_SERVICES:
+            service_pause(service_)
+    else:
+        for service_ in AWS_COMPAT_SERVICES:
+            service_resume(service_)

unit_tests/test_actions.py

@@ -14,9 +14,21 @@
 
 import mock
 
+from test_utils import (
+    CharmTestCase,
+    get_default_config,
+)
+
+__default_config = get_default_config()
+# NOTE(freyes): the default 'distro' makes the test suite behave different
+# depending on where it's being executed
+__default_config['openstack-origin'] = ''
+
 with mock.patch('charmhelpers.core.hookenv.config') as config:
     with mock.patch('charmhelpers.contrib.openstack.utils.get_os_codename_package'):  # noqa
-        config.return_value = 'nova'
+        # this makes the config behave more similar to the real config()
+        config.side_effect = lambda k: __default_config[k]
+
         import nova_cc_utils as utils  # noqa
 
 # Need to do some early patching to get the module loaded.
@@ -36,10 +48,6 @@ with mock.patch('nova_cc_utils.guard_map') as gmap:
 utils.register_configs = _reg
 utils.restart_map = _map
 
-from test_utils import (
-    CharmTestCase
-)
-
 TO_PATCH = [
 ]
 

unit_tests/test_actions_git_reinstall.py

@@ -60,8 +60,10 @@ class TestnovaAPIActions(CharmTestCase):
     @patch.object(git_reinstall, 'action_fail')
     @patch.object(git_reinstall, 'git_install')
     @patch.object(git_reinstall, 'config_changed')
-    def test_git_reinstall(self, config_changed, git_install, action_fail,
+    @patch('charmhelpers.contrib.openstack.utils.config')
-                           action_set):
+    def test_git_reinstall(self, _config, config_changed, git_install,
+                           action_fail, action_set):
+        _config.side_effect = self.test_config.get
         self.test_config.set('openstack-origin-git', openstack_origin_git)
 
         git_reinstall.git_reinstall()

unit_tests/test_nova_cc_contexts.py

@@ -126,6 +126,7 @@ class NovaComputeContextTests(CharmTestCase):
         self.assertFalse('neutron_url' in ctxt)
 
     @mock.patch.object(neutron, 'network_manager')
+    @mock.patch('charmhelpers.contrib.openstack.context.unit_get')
     @mock.patch('charmhelpers.contrib.hahelpers.cluster.https')
     @mock.patch('charmhelpers.contrib.openstack.context.kv')
     @mock.patch('charmhelpers.contrib.openstack.context.'
@@ -138,9 +139,10 @@ class NovaComputeContextTests(CharmTestCase):
     def test_haproxy_context(self, mock_relation_ids, mock_get_ipv6_addr,
                              mock_local_unit, mock_get_netmask_for_address,
                              mock_get_address_in_network, mock_kv, mock_https,
-                             mock_network_manager):
+                             mock_unit_get, mock_network_manager):
-        mock_network_manager.return_value = 'neutron'
         mock_https.return_value = False
+        mock_unit_get.return_value = '127.0.0.1'
+        mock_network_manager.return_value = 'neutron'
         ctxt = context.HAProxyContext()()
         self.assertEqual(ctxt['service_ports']['nova-api-os-compute'],
                          [8774, 8764])
@@ -293,11 +295,15 @@ class NovaComputeContextTests(CharmTestCase):
         self.assertEqual(ctxt['html5proxy_base_url'],
                          'https://10.5.0.1:6082/spice_auto.html')
 
+    @mock.patch('charmhelpers.contrib.openstack.ip.unit_get')
+    @mock.patch('charmhelpers.contrib.hahelpers.cluster.relation_ids')
     @mock.patch('charmhelpers.core.hookenv.local_unit')
     @mock.patch('charmhelpers.contrib.openstack.context.config')
-    def test_nova_config_context(self, mock_config, local_unit):
+    def test_nova_config_context(self, mock_config, local_unit,
+                                 mock_relation_ids, mock_unit_get):
         local_unit.return_value = 'nova-cloud-controller/0'
         mock_config.side_effect = self.test_config.get
+        mock_unit_get.return_value = '127.0.0.1'
         ctxt = context.NovaConfigContext()()
         self.assertEqual(ctxt['scheduler_default_filters'],
                          self.config('scheduler-default-filters'))

unit_tests/test_nova_cc_hooks.py

@@ -164,17 +164,23 @@ class NovaCCHooksTests(CharmTestCase):
         self.assertTrue(self.disable_services.called)
         self.cmd_all_services.assert_called_with('stop')
 
+    @patch.object(utils, 'service_resume')
+    @patch.object(utils, 'config')
     @patch.object(hooks, 'filter_installed_packages')
     @patch.object(hooks, 'configure_https')
-    def test_config_changed_no_upgrade(self, conf_https, mock_filter_packages):
+    def test_config_changed_no_upgrade(self, conf_https, mock_filter_packages,
+                                       utils_config, mock_service_resume):
+        utils_config.side_effect = self.test_config.get
+        self.test_config.set('console-access-protocol', 'dummy')
         self.git_install_requested.return_value = False
         self.openstack_upgrade_available.return_value = False
         hooks.config_changed()
         self.assertTrue(self.save_script_rc.called)
         mock_filter_packages.assert_called_with([])
 
+    @patch.object(utils, 'service_resume')
     @patch.object(hooks, 'configure_https')
-    def test_config_changed_git(self, configure_https):
+    def test_config_changed_git(self, configure_https, mock_service_resume):
         self.git_install_requested.return_value = True
         repo = 'cloud:trusty-juno'
         openstack_origin_git = {
@@ -196,6 +202,10 @@ class NovaCCHooksTests(CharmTestCase):
         self.git_install.assert_called_with(projects_yaml)
         self.assertFalse(self.do_openstack_upgrade.called)
 
+    @patch.object(utils, 'service_resume')
+    @patch('charmhelpers.contrib.openstack.ip.unit_get')
+    @patch('charmhelpers.contrib.hahelpers.cluster.relation_ids')
+    @patch.object(utils, 'config')
     @patch.object(hooks, 'db_joined')
     @patch.object(hooks, 'filter_installed_packages')
     @patch('charmhelpers.contrib.openstack.ip.service_name',
@@ -206,11 +216,18 @@ class NovaCCHooksTests(CharmTestCase):
     @patch.object(hooks, 'configure_https')
     def test_config_changed_with_upgrade(self, conf_https, neutron_api_joined,
                                          identity_joined, cluster_joined,
-                                         mock_filter_packages, db_joined):
+                                         mock_filter_packages, db_joined,
+                                         utils_config, mock_relids,
+                                         mock_unit_get,
+                                         mock_service_resume):
         self.git_install_requested.return_value = False
         self.openstack_upgrade_available.return_value = True
         self.relation_ids.return_value = ['generic_rid']
         _zmq_joined = self.patch('zeromq_configuration_relation_joined')
+        utils_config.side_effect = self.test_config.get
+        self.test_config.set('console-access-protocol', 'dummy')
+        mock_relids.return_value = []
+        mock_unit_get.return_value = '127.0.0.1'
         hooks.config_changed()
         self.assertTrue(self.do_openstack_upgrade.called)
         self.assertTrue(neutron_api_joined.called)
@@ -221,12 +238,14 @@ class NovaCCHooksTests(CharmTestCase):
         self.assertTrue(self.save_script_rc.called)
         mock_filter_packages.assert_called_with([])
 
+    @patch.object(utils, 'service_resume')
     @patch.object(hooks, 'filter_installed_packages')
     @patch.object(hooks, 'configure_https')
     @patch.object(hooks, 'compute_changed')
     def test_config_changed_region_change(self, mock_compute_changed,
                                           mock_config_https,
-                                          mock_filter_packages):
+                                          mock_filter_packages,
+                                          mock_service_resume):
         self.git_install_requested.return_value = False
         self.openstack_upgrade_available.return_value = False
         self.config_value_changed.return_value = True
@@ -942,12 +961,14 @@ class NovaCCHooksTests(CharmTestCase):
             call(**args),
         ])
 
+    @patch.object(utils, 'service_pause')
     @patch.object(hooks, 'filter_installed_packages')
     @patch('nova_cc_hooks.configure_https')
     @patch('nova_cc_utils.config')
     def test_config_changed_single_consoleauth(self, mock_config,
                                                mock_configure_https,
-                                               mock_filter_packages):
+                                               mock_filter_packages,
+                                               mock_service_pause):
         self.config_value_changed.return_value = False
         self.git_install_requested.return_value = False
         config.return_value = 'novnc'

unit_tests/test_nova_cc_utils.py

@@ -15,11 +15,20 @@
 from collections import OrderedDict
 from mock import patch, MagicMock, call
 
-with patch('charmhelpers.core.hookenv.config'):
+from test_utils import (
-    with patch('charmhelpers.contrib.openstack.utils.get_os_codename_package'):  # noqa
+    CharmTestCase,
-        import nova_cc_utils as utils
+    get_default_config,
+    patch_open,
+)
 
-from test_utils import CharmTestCase, patch_open
+__default_config = get_default_config()
+
+with patch('charmhelpers.core.hookenv.config') as config:
+    with patch('charmhelpers.contrib.openstack.utils.get_os_codename_package'):  # noqa
+        # this makes the config behave more similar to the real config()
+        config.side_effect = lambda k: __default_config[k]
+
+        import nova_cc_utils as utils
 
 TO_PATCH = [
     'apt_update',
@@ -1074,3 +1083,43 @@ class NovaCCUtilsTests(CharmTestCase):
             asf.assert_called_once_with('some-config')
             # ports=None whilst port checks are disabled.
             f.assert_called_once_with('assessor', services='s1', ports=None)
+
+    @patch.object(utils, 'service_pause')
+    @patch.object(utils, 'service_resume')
+    @patch.object(utils, 'config')
+    @patch.object(utils, 'filter_installed_packages')
+    def test_disable_aws_compat_services_uinstalled(self,
+                                                    filter_installed_packages,
+                                                    config, service_resume,
+                                                    service_pause):
+        filter_installed_packages.return_value = utils.AWS_COMPAT_SERVICES
+        utils.update_aws_compat_services()
+        config.assert_not_called()
+        service_pause.assert_not_called()
+        service_resume.assert_not_called()
+
+    @patch.object(utils, 'service_pause')
+    @patch.object(utils, 'service_resume')
+    @patch.object(utils, 'config')
+    @patch.object(utils, 'filter_installed_packages')
+    def test_disable_aws_compat_services_true(self, filter_installed_packages,
+                                              config, s_resume, s_pause):
+        filter_installed_packages.return_value = []
+        config.return_value = True
+        utils.update_aws_compat_services()
+
+        s_resume.assert_not_called()
+        s_pause.assert_has_calls([call(s) for s in utils.AWS_COMPAT_SERVICES])
+
+    @patch.object(utils, 'service_pause')
+    @patch.object(utils, 'service_resume')
+    @patch.object(utils, 'config')
+    @patch.object(utils, 'filter_installed_packages')
+    def test_disable_aws_compat_services_false(self, filter_installed_packages,
+                                               config, s_resume, s_pause):
+        filter_installed_packages.return_value = []
+        config.return_value = False
+        utils.update_aws_compat_services()
+
+        s_resume.assert_has_calls([call(s) for s in utils.AWS_COMPAT_SERVICES])
+        s_pause.assert_not_called()