Drop apache+mod-fastcgi support

The ceph project has focussed on running the RADOS Gateway using
the embedded civetweb engine for many cycles now; mod-fastcgi is
buggy and no longer provided in Ubuntu as of 17.04, so switch to
always running in embedded mode.

Existing installs will be upgraded to run in this mode, purging
apache related packaging and configuration.

Change-Id: I90e6c047d78de2b0ebf8c24bd2f2d6d1bfbd5c5d
Closes-Bug: 1657370

config.yaml

@@ -75,26 +75,6 @@ options:
     default: False
     description: |
       If set to True, supporting services will log to syslog.
-  use-ceph-optimised-packages:
-    type: boolean
-    default: false
-    description: |
-      By default apache2 and libapache2-mod-fastcgi will be installed from the
-      Ubuntu archives. This option allows for an alternate ceph.com install
-      source which contains patched versions with added support for HTTP
-      100-continue. See the following page for more info:
-
-      http://ceph.com/docs/dumpling/radosgw/manual-install/#continue-support
-  use-embedded-webserver:
-    type: boolean
-    default: false
-    description: |
-      Newer versions of the Ceph RADOS Gateway support use of an embedded web
-      container instead of Apache + mod-fastcgi, avoiding some of the nuances
-      of using the stock mod-fastcgi packages from Ubuntu.
-
-      Enable this option to disable use of Apache and enable the embedded
-      web container feature.
   dns-ha:
     type: boolean
     default: False

hooks/ceph_radosgw_context.py

@@ -16,9 +16,7 @@ import os
 import re
 import socket
 import tempfile
-import glob
 import shutil
-import subprocess
 
 from charmhelpers.contrib.openstack import context
 from charmhelpers.contrib.hahelpers.cluster import (
@@ -34,7 +32,6 @@ from charmhelpers.core.hookenv import (
     relation_ids,
     related_units,
     relation_get,
-    status_set,
 )
 from charmhelpers.contrib.network.ip import (
     format_ipv6_addr,
@@ -44,55 +41,6 @@ from charmhelpers.contrib.network.ip import (
 from charmhelpers.contrib.storage.linux.ceph import CephConfContext
 
 
-def is_apache_24():
-    if os.path.exists('/etc/apache2/conf-available'):
-        return True
-    else:
-        return False
-
-
-class ApacheContext(context.OSContextGenerator):
-    interfaces = ['http']
-    service_namespace = 'ceph-radosgw'
-
-    def __call__(self):
-        ctxt = {}
-        if config('use-embedded-webserver'):
-            log("Skipping ApacheContext since we are using the embedded "
-                "webserver")
-            return {}
-
-        status_set('maintenance', 'configuring apache')
-
-        src = 'files/www/*'
-        dst = '/var/www/'
-        log("Installing www scripts", level=DEBUG)
-        try:
-            for x in glob.glob(src):
-                shutil.copy(x, dst)
-        except IOError as e:
-            log("Error copying files from '%s' to '%s': %s" % (src, dst, e),
-                level=WARNING)
-
-        try:
-            subprocess.check_call(['a2enmod', 'fastcgi'])
-            subprocess.check_call(['a2enmod', 'rewrite'])
-        except subprocess.CalledProcessError as e:
-            log("Error enabling apache modules - %s" % e, level=WARNING)
-
-        try:
-            if is_apache_24():
-                subprocess.check_call(['a2dissite', '000-default'])
-            else:
-                subprocess.check_call(['a2dissite', 'default'])
-        except subprocess.CalledProcessError as e:
-            log("Error disabling apache sites - %s" % e, level=WARNING)
-
-        ctxt['hostname'] = socket.gethostname()
-        ctxt['port'] = determine_api_port(config('port'), singlenode_mode=True)
-        return ctxt
-
-
 class HAProxyContext(context.HAProxyContext):
 
     def __call__(self):
@@ -220,7 +168,6 @@ class MonContext(context.OSContextGenerator):
             'hostname': host,
             'old_auth': cmp_pkgrevno('radosgw', "0.51") < 0,
             'use_syslog': str(config('use-syslog')).lower(),
-            'embedded_webserver': config('use-embedded-webserver'),
             'loglevel': config('loglevel'),
             'port': port,
             'ipv6': config('prefer-ipv6')
@@ -232,13 +179,6 @@ class MonContext(context.OSContextGenerator):
         if all([os.path.isfile(p) for p in paths]):
             ctxt['cms'] = True
 
-        if (config('use-ceph-optimised-packages') and
-                not config('use-embedded-webserver')):
-            ctxt['disable_100_continue'] = False
-        else:
-            # NOTE: currently only applied if NOT using embedded webserver
-            ctxt['disable_100_continue'] = True
-
         # NOTE(dosaboy): these sections must correspond to what is supported in
         #                the config template.
         sections = ['global', 'client.radosgw.gateway']

hooks/hooks.py

@@ -30,7 +30,6 @@ from charmhelpers.core.hookenv import (
     relation_set,
     log,
     DEBUG,
-    WARNING,
     Hooks, UnregisteredHookError,
     status_set,
 )
@@ -39,8 +38,8 @@ from charmhelpers.fetch import (
     apt_install,
     apt_purge,
     add_source,
+    filter_installed_packages,
 )
-from charmhelpers.core.host import lsb_release
 from charmhelpers.payload.execd import execd_preinstall
 from charmhelpers.core.host import cmp_pkgrevno
 from charmhelpers.contrib.network.ip import (
@@ -86,21 +85,6 @@ CONFIGS = register_configs()
 NSS_DIR = '/var/lib/ceph/nss'
 
 
-def install_ceph_optimised_packages():
-    """Inktank provides patched/optimised packages for HTTP 100-continue
-    support that does has not yet been ported to upstream. These can
-    optionally be installed from ceph.com archives.
-    """
-    prolog = "http://gitbuilder.ceph.com/"
-    epilog = "-x86_64-basic/ref/master"
-    rel = lsb_release()['DISTRIB_CODENAME']
-    fastcgi_source = "%slibapache-mod-fastcgi-deb-%s%s" % (prolog, rel, epilog)
-    apache_source = "%sapache2-deb-%s%s" % (prolog, rel, epilog)
-
-    for source in [fastcgi_source, apache_source]:
-        add_source(source, key='6EAEAE2203C3951A')
-
-
 PACKAGES = [
     'haproxy',
     'libnss3-tools',
@@ -119,18 +103,13 @@ APACHE_PACKAGES = [
 
 
 def install_packages():
-    status_set('maintenance', 'Installing apt packages')
     add_source(config('source'), config('key'))
-    if (config('use-ceph-optimised-packages') and
-            not config('use-embedded-webserver')):
-        install_ceph_optimised_packages()
-
     apt_update(fatal=True)
+    pkgs = filter_installed_packages(PACKAGES)
+    if pkgs:
+        status_set('maintenance', 'Installing radosgw packages')
         apt_install(PACKAGES, fatal=True)
-    if config('use-embedded-webserver'):
     apt_purge(APACHE_PACKAGES)
-    else:
-        apt_install(APACHE_PACKAGES, fatal=True)
 
 
 @hooks.hook('install.real')
@@ -166,15 +145,6 @@ def config_changed():
 
     CONFIGS.write_all()
 
-    if not config('use-embedded-webserver'):
-        try:
-            subprocess.check_call(['a2ensite', 'rgw'])
-        except subprocess.CalledProcessError as e:
-            log("Error enabling apache module 'rgw' - %s" % e, level=WARNING)
-
-        # Ensure started but do a soft reload
-        subprocess.call(['service', 'apache2', 'start'])
-        subprocess.call(['service', 'apache2', 'reload'])
     update_nrpe_config()
 
 

hooks/utils.py

@@ -23,7 +23,6 @@ from copy import deepcopy
 import ceph_radosgw_context
 
 from charmhelpers.core.hookenv import (
-    config,
     log,
     DEBUG,
     INFO,
@@ -96,9 +95,6 @@ TEMPLATES_DIR = 'templates'
 TEMPLATES = 'templates/'
 HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
 CEPH_CONF = '/etc/ceph/ceph.conf'
-APACHE_CONF = '/etc/apache2/sites-available/rgw'
-APACHE_24_CONF = '/etc/apache2/sites-available/rgw.conf'
-APACHE_PORTS_CONF = '/etc/apache2/ports.conf'
 
 VERSION_PACKAGE = 'radosgw'
 
@@ -108,18 +104,6 @@ BASE_RESOURCE_MAP = OrderedDict([
                      ceph_radosgw_context.HAProxyContext()],
         'services': ['haproxy'],
     }),
-    (APACHE_CONF, {
-        'contexts': [ceph_radosgw_context.ApacheContext()],
-        'services': ['apache2'],
-    }),
-    (APACHE_24_CONF, {
-        'contexts': [ceph_radosgw_context.ApacheContext()],
-        'services': ['apache2'],
-    }),
-    (APACHE_PORTS_CONF, {
-        'contexts': [ceph_radosgw_context.ApacheContext()],
-        'services': ['apache2'],
-    }),
     (CEPH_CONF, {
         'contexts': [ceph_radosgw_context.MonContext()],
         'services': ['radosgw'],
@@ -140,16 +124,6 @@ def resource_map():
 
     These will be managed for a single hook execution.
     """
-    if not config('use-embedded-webserver'):
-        if os.path.exists('/etc/apache2/conf-available'):
-            BASE_RESOURCE_MAP.pop(APACHE_CONF)
-        else:
-            BASE_RESOURCE_MAP.pop(APACHE_24_CONF)
-    else:
-        BASE_RESOURCE_MAP.pop(APACHE_CONF)
-        BASE_RESOURCE_MAP.pop(APACHE_24_CONF)
-        BASE_RESOURCE_MAP.pop(APACHE_PORTS_CONF)
-
     resource_map = deepcopy(BASE_RESOURCE_MAP)
     return resource_map
 

templates/ceph.conf

@@ -28,13 +28,7 @@ rgw init timeout = 1200
 keyring = /etc/ceph/keyring.rados.gateway
 rgw socket path = /tmp/radosgw.sock
 log file = /var/log/ceph/radosgw.log
-{% if embedded_webserver %}
 rgw frontends = civetweb port={{ port }}
-{% elif disable_100_continue %}
-# Turn off 100-continue optimization as stock mod_fastcgi
-# does not support it
-rgw print continue = false
-{% endif %}
 {% if auth_type == 'keystone' %}
 rgw keystone url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/
 rgw keystone admin token = {{ admin_token }}

templates/ports.conf

@@ -1,11 +0,0 @@
-Listen {{ port }}
-
-<IfModule ssl_module>
-	Listen 443
-</IfModule>
-
-<IfModule mod_gnutls.c>
-	Listen 443
-</IfModule>
-
-# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

templates/rgw

@@ -1,25 +0,0 @@
-<IfModule mod_fastcgi.c>
-	FastCgiExternalServer /var/www/s3gw.fcgi -socket /tmp/radosgw.sock
-</IfModule>
-
-<VirtualHost *:{{ port }}>
-	ServerName {{ hostname }}
-	ServerAdmin ceph@ubuntu.com
-	DocumentRoot /var/www
-	RewriteEngine On
-	RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1&params=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
-	<IfModule mod_fastcgi.c>
-		<Directory /var/www>
-			Options +ExecCGI
-			AllowOverride All
-			SetHandler fastcgi-script
-			Order allow,deny
-			Allow from all
-			AuthBasicAuthoritative Off
-		</Directory>
-	</IfModule>
-	AllowEncodedSlashes On
-	ErrorLog /var/log/apache2/error.log
-	CustomLog /var/log/apache2/access.log combined
-	ServerSignature Off
-</VirtualHost>

templates/rgw.conf

@@ -1,25 +0,0 @@
-<IfModule mod_fastcgi.c>
-	FastCgiExternalServer /var/www/s3gw.fcgi -socket /tmp/radosgw.sock
-</IfModule>
-
-<VirtualHost *:{{ port }}>
-	ServerName {{ hostname }}
-	ServerAdmin ceph@ubuntu.com
-	DocumentRoot /var/www
-	RewriteEngine On
-	RewriteRule ^/([a-zA-Z0-9-_.]*)([/]?.*) /s3gw.fcgi?page=$1&params=$2&%{QUERY_STRING} [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
-	<IfModule mod_fastcgi.c>
-		<Directory /var/www>
-			Options +ExecCGI
-			AllowOverride All
-			SetHandler fastcgi-script
-			Order allow,deny
-			Allow from all
-			AuthBasicAuthoritative Off
-		</Directory>
-	</IfModule>
-	AllowEncodedSlashes On
-	ErrorLog /var/log/apache2/error.log
-	CustomLog /var/log/apache2/access.log combined
-	ServerSignature Off
-</VirtualHost>

tests/basic_deployment.py

@@ -115,13 +115,11 @@ class CephRadosGwBasicDeployment(OpenStackAmuletDeployment):
             'ephemeral-unmount': '/mnt',
             'osd-devices': '/dev/vdb /srv/ceph'
         }
-        radosgw_config = {"use-embedded-webserver": True}
 
         configs = {'keystone': keystone_config,
                    'percona-cluster': pxc_config,
                    'cinder': cinder_config,
-                   'ceph': ceph_config,
+                   'ceph': ceph_config}
-                   'ceph-radosgw': radosgw_config}
         super(CephRadosGwBasicDeployment, self)._configure_services(configs)
 
     def _run_action(self, unit_id, action, *args):

unit_tests/test_ceph_radosgw_context.py

@@ -28,7 +28,6 @@ TO_PATCH = [
     'related_units',
     'cmp_pkgrevno',
     'socket',
-    'is_apache_24',
 ]
 
 
@@ -182,8 +181,6 @@ class MonContextTest(CharmTestCase):
         self.related_units.return_value = ['ceph/0', 'ceph/1', 'ceph/2']
         expect = {
             'auth_supported': 'cephx',
-            'embedded_webserver': False,
-            'disable_100_continue': True,
             'hostname': 'testhost',
             'mon_hosts': '10.5.4.1:6789 10.5.4.2:6789 10.5.4.3:6789',
             'old_auth': False,
@@ -231,8 +228,6 @@ class MonContextTest(CharmTestCase):
         self.related_units.return_value = ['ceph/0', 'ceph/1', 'ceph/2']
         expect = {
             'auth_supported': 'none',
-            'embedded_webserver': False,
-            'disable_100_continue': True,
             'hostname': 'testhost',
             'mon_hosts': '10.5.4.1:6789 10.5.4.2:6789 10.5.4.3:6789',
             'old_auth': False,
@@ -262,8 +257,6 @@ class MonContextTest(CharmTestCase):
         self.related_units.return_value = ['ceph/0', 'ceph/1', 'ceph/2']
         expect = {
             'auth_supported': 'cephx',
-            'embedded_webserver': False,
-            'disable_100_continue': True,
             'hostname': 'testhost',
             'mon_hosts': '10.5.4.1:6789 10.5.4.2:6789 10.5.4.3:6789',
             'old_auth': False,

unit_tests/test_hooks.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 from mock import (
-    call,
     patch,
 )
 
@@ -42,7 +41,6 @@ TO_PATCH = [
     'get_iface_for_address',
     'get_netmask_for_address',
     'log',
-    'lsb_release',
     'open_port',
     'os',
     'relation_ids',
@@ -67,50 +65,10 @@ class CephRadosGWTests(CharmTestCase):
         self.test_config.set('key', 'secretkey')
         self.test_config.set('use-syslog', False)
 
-    def test_install_ceph_optimised_packages(self):
-        self.lsb_release.return_value = {'DISTRIB_CODENAME': 'vivid'}
-        fastcgi_source = (
-            'http://gitbuilder.ceph.com/'
-            'libapache-mod-fastcgi-deb-vivid-x86_64-basic/ref/master')
-        apache_source = (
-            'http://gitbuilder.ceph.com/'
-            'apache2-deb-vivid-x86_64-basic/ref/master')
-        calls = [
-            call(fastcgi_source, key='6EAEAE2203C3951A'),
-            call(apache_source, key='6EAEAE2203C3951A'),
-        ]
-        ceph_hooks.install_ceph_optimised_packages()
-        self.add_source.assert_has_calls(calls)
-
     def test_install_packages(self):
-        self.test_config.set('use-ceph-optimised-packages', '')
         ceph_hooks.install_packages()
         self.add_source.assert_called_with('distro', 'secretkey')
         self.assertTrue(self.apt_update.called)
-        self.apt_install.assert_called_with(['libapache2-mod-fastcgi',
-                                             'apache2'], fatal=True)
-
-    def test_install_optimised_packages_no_embedded(self):
-        self.test_config.set('use-ceph-optimised-packages', True)
-        self.test_config.set('use-embedded-webserver', False)
-        _install_packages = self.patch('install_ceph_optimised_packages')
-        ceph_hooks.install_packages()
-        self.add_source.assert_called_with('distro', 'secretkey')
-        self.assertTrue(self.apt_update.called)
-        self.assertTrue(_install_packages.called)
-        self.apt_install.assert_called_with(['libapache2-mod-fastcgi',
-                                             'apache2'], fatal=True)
-
-    def test_install_optimised_packages_embedded(self):
-        self.test_config.set('use-ceph-optimised-packages', True)
-        self.test_config.set('use-embedded-webserver', True)
-        _install_packages = self.patch('install_ceph_optimised_packages')
-        ceph_hooks.install_packages()
-        self.add_source.assert_called_with('distro', 'secretkey')
-        self.assertTrue(self.apt_update.called)
-        self.assertFalse(_install_packages.called)
-        self.apt_install.assert_called_with(ceph_hooks.PACKAGES,
-                                            fatal=True)
         self.apt_purge.assert_called_with(['libapache2-mod-fastcgi',
                                            'apache2'])