[yolanda] Add postgresql support

Refactor templates for includes

Makefile

@@ -1,11 +1,10 @@
 #!/usr/bin/make
 
 lint:
-	@echo -n "Running flake8 tests: "
+	@echo "Running flake8 tests: "
-	@flake8 --exclude hooks/charmhelpers hooks
+	@flake8 --exclude hooks/charmhelpers hooks unit_tests
-	@flake8 unit_tests
 	@echo "OK"
-	@echo -n "Running charm proof: "
+	@echo "Running charm proof: "
 	@charm proof
 	@echo "OK"
 

hooks/charmhelpers/contrib/openstack/context.py

@@ -147,7 +147,8 @@ class SharedDBContext(OSContextGenerator):
                     'database_host': rdata.get('db_host'),
                     'database': self.database,
                     'database_user': self.user,
-                    'database_password': rdata.get(password_setting)
+                    'database_password': rdata.get(password_setting),
+                    'database_type': 'mysql'
                 }
                 if context_complete(ctxt):
                     db_ssl(rdata, ctxt, self.ssl_dir)
@@ -155,6 +156,35 @@ class SharedDBContext(OSContextGenerator):
         return {}
 
 
+class PostgresqlDBContext(OSContextGenerator):
+    interfaces = ['pgsql-db']
+
+    def __init__(self, database=None):
+        self.database = database
+
+    def __call__(self):
+        self.database = self.database or config('database')
+        if self.database is None:
+            log('Could not generate postgresql_db context. '
+                'Missing required charm config options. '
+                '(database name)')
+            raise OSContextError
+        ctxt = {}
+
+        for rid in relation_ids(self.interfaces[0]):
+            for unit in related_units(rid):
+                ctxt = {
+                    'database_host': relation_get('host', rid=rid, unit=unit),
+                    'database': self.database,
+                    'database_user': relation_get('user', rid=rid, unit=unit),
+                    'database_password': relation_get('password', rid=rid, unit=unit),
+                    'database_type': 'postgresql',
+                }
+                if context_complete(ctxt):
+                    return ctxt
+        return {}
+
+
 def db_ssl(rdata, ctxt, ssl_dir):
     if 'ssl_ca' in rdata and ssl_dir:
         ca_path = os.path.join(ssl_dir, 'db-client.ca')

hooks/glance_contexts.py

@@ -1,8 +1,7 @@
 from charmhelpers.core.hookenv import (
     is_relation_made,
     relation_ids,
-    service_name,
+    service_name
-    config
 )
 
 from charmhelpers.contrib.openstack.context import (

hooks/glance_relations.py

@@ -22,7 +22,9 @@ from charmhelpers.core.hookenv import (
     config,
     Hooks,
     log as juju_log,
+    ERROR,
     open_port,
+    is_relation_made,
     relation_get,
     relation_set,
     relation_ids,
@@ -32,8 +34,7 @@ from charmhelpers.core.hookenv import (
 
 from charmhelpers.core.host import (
     restart_on_change,
-    service_stop,
+    service_stop
-    mkdir
 )
 
 from charmhelpers.fetch import apt_install, apt_update
@@ -79,10 +80,29 @@ def install_hook():
 
 @hooks.hook('shared-db-relation-joined')
 def db_joined():
+    if is_relation_made('pgsql-db'):
+        # error, postgresql is used
+        e = ('Attempting to associate a mysql database when there is already '
+             'associated a postgresql one')
+        juju_log(e, level=ERROR)
+        raise Exception(e)
+
     relation_set(database=config('database'), username=config('database-user'),
                  hostname=unit_get('private-address'))
 
 
+@hooks.hook('pgsql-db-relation-joined')
+def pgsql_db_joined():
+    if is_relation_made('shared-db'):
+        # raise error
+        e = ('Attempting to associate a postgresql database when'
+             ' there is already associated a mysql one')
+        juju_log(e, level=ERROR)
+        raise Exception(e)
+
+    relation_set(database=config('database'))
+
+
 @hooks.hook('shared-db-relation-changed')
 @restart_on_change(restart_map())
 def db_changed():
@@ -108,6 +128,31 @@ def db_changed():
         migrate_database()
 
 
+@hooks.hook('pgsql-db-relation-changed')
+@restart_on_change(restart_map())
+def pgsql_db_changed():
+    rel = get_os_codename_package("glance-common")
+
+    if 'pgsql-db' not in CONFIGS.complete_contexts():
+        juju_log('pgsql-db relation incomplete. Peer not ready?')
+        return
+
+    CONFIGS.write(GLANCE_REGISTRY_CONF)
+    # since folsom, a db connection setting in glance-api.conf is required.
+    if rel != "essex":
+        CONFIGS.write(GLANCE_API_CONF)
+
+    if eligible_leader(CLUSTER_RES):
+        if rel == "essex":
+            status = call(['glance-manage', 'db_version'])
+            if status != 0:
+                juju_log('Setting version_control to 0')
+                check_call(["glance-manage", "version_control", "0"])
+
+        juju_log('Cluster leader, performing db sync')
+        migrate_database()
+
+
 @hooks.hook('image-service-relation-joined')
 def image_service_joined(relation_id=None):
     if not eligible_leader(CLUSTER_RES):
@@ -216,10 +261,10 @@ def config_changed():
     open_port(9292)
     configure_https()
 
-    #env_vars = {'OPENSTACK_PORT_MCASTPORT': config("ha-mcastport"),
+    # env_vars = {'OPENSTACK_PORT_MCASTPORT': config("ha-mcastport"),
     #            'OPENSTACK_SERVICE_API': "glance-api",
     #            'OPENSTACK_SERVICE_REGISTRY': "glance-registry"}
-    #save_script_rc(**env_vars)
+    # save_script_rc(**env_vars)
 
 
 @hooks.hook('cluster-relation-changed')
@@ -243,7 +288,7 @@ def ha_relation_joined():
     vip_iface = config("vip_iface")
     vip_cidr = config("vip_cidr")
 
-    #if vip and vip_iface and vip_cidr and \
+    # if vip and vip_iface and vip_cidr and \
     #    corosync_bindiface and corosync_mcastport:
 
     resources = {
@@ -289,7 +334,8 @@ def ha_relation_changed():
 @hooks.hook('ceph-relation-broken',
             'identity-service-relation-broken',
             'object-store-relation-broken',
-            'shared-db-relation-broken')
+            'shared-db-relation-broken',
+            'pgsql-db-relation-broken')
 def relation_broken():
     CONFIGS.write_all()
 

hooks/glance_utils.py

@@ -46,7 +46,7 @@ CLUSTER_RES = "res_glance_vip"
 
 PACKAGES = [
     "apache2", "glance", "python-mysqldb", "python-swift",
-    "python-keystone", "uuid", "haproxy", ]
+    "python-psycopg2", "python-keystone", "uuid", "haproxy", ]
 
 SERVICES = [
     "glance-api", "glance-registry", ]
@@ -70,12 +70,14 @@ CONF_DIR = "/etc/glance"
 
 TEMPLATES = 'templates/'
 
+
 def ceph_config_file():
     return CHARM_CEPH_CONF.format(service_name())
 
 CONFIG_FILES = OrderedDict([
     (GLANCE_REGISTRY_CONF, {
         'hook_contexts': [context.SharedDBContext(ssl_dir=GLANCE_CONF_DIR),
+                          context.PostgresqlDBContext(),
                           context.IdentityServiceContext(),
                           context.SyslogContext()],
         'services': ['glance-registry']
@@ -83,6 +85,7 @@ CONFIG_FILES = OrderedDict([
     (GLANCE_API_CONF, {
         'hook_contexts': [context.SharedDBContext(ssl_dir=GLANCE_CONF_DIR),
                           context.AMQPContext(ssl_dir=GLANCE_CONF_DIR),
+                          context.PostgresqlDBContext(),
                           context.IdentityServiceContext(),
                           glance_contexts.CephGlanceContext(),
                           glance_contexts.ObjectStoreContext(),
@@ -117,6 +120,7 @@ CONFIG_FILES = OrderedDict([
     })
 ])
 
+
 def register_configs():
     # Register config files with their respective contexts.
     # Regstration of some configs may not be required depending on

hooks/pgsql-db-relation-broken

@@ -0,0 +1 @@
+glance_relations.py

hooks/pgsql-db-relation-changed

@@ -0,0 +1 @@
+glance_relations.py

hooks/pgsql-db-relation-joined

@@ -0,0 +1 @@
+glance_relations.py

metadata.yaml

@@ -14,6 +14,8 @@ provides:
 requires:
   shared-db:
     interface: mysql-shared
+  pgsql-db:
+    interface: pgsql
   amqp:
     interface: rabbitmq
   object-store:

setup.cfg

@@ -0,0 +1,5 @@
+[nosetests]
+verbosity=2
+with-coverage=1
+cover-erase=1
+cover-package=hooks

templates/folsom/glance-api.conf

@@ -19,31 +19,18 @@ bind_port = 9292
 {% endif %}
 log_file = /var/log/glance/api.log
 backlog = 4096
-{% if database_host %}
+
-sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %}
+{% include "parts/database" %}
-{% else %}
+
-sql_connection = sqlite:////var/lib/glance/glance.sqlite
-{% endif %}
 sql_idle_timeout = 3600
 workers = 1
 registry_host = 0.0.0.0
 registry_port = 9191
 registry_client_protocol = http
 
-{% if rabbitmq_host -%}
+{% include "parts/rabbitmq-single" %}
+
 notifier_strategy = rabbit
-rabbit_host = {{ rabbitmq_host }}
-rabbit_userid = {{ rabbitmq_user }}
-rabbit_password = {{ rabbitmq_password }}
-rabbit_virtual_host = {{ rabbitmq_virtual_host }}
-{% if rabbit_ssl_port %}
-rabbit_use_ssl=True
-rabbit_port={{ rabbit_ssl_port }}
-{% if rabbit_ssl_ca %}
-kombu_ssl_ca_certs={{rabbit_ssl_ca}}
-{% endif %}
-{% endif %}
-{% endif -%}
 filesystem_store_datadir = /var/lib/glance/images/
 
 {% if swift_store %}

templates/glance-registry.conf

@@ -6,13 +6,12 @@ bind_host = 0.0.0.0
 bind_port = 9191
 log_file = /var/log/glance/registry.log
 backlog = 4096
-{% if database_host %}
+
-sql_connection = mysql://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %}
+{% include "parts/database" %}
-{% endif %}
+
 sql_idle_timeout = 3600
 api_limit_max = 1000
 limit_param_default = 25
-use_syslog = False
 
 {% if auth_host %}
 [paste_deploy]

templates/parts/database

@@ -0,0 +1,3 @@
+{% if database_host -%}
+sql_connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %}
+{% endif -%}

templates/parts/rabbitmq

@@ -0,0 +1,21 @@
+{% if rabbitmq_host or rabbitmq_hosts -%}
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_virtual_host = {{ rabbitmq_virtual_host }}
+rabbit_password = {{ rabbitmq_password }}
+{% if rabbitmq_hosts -%}
+rabbit_hosts = {{ rabbitmq_hosts }}
+{% if rabbitmq_ha_queues -%}
+rabbit_ha_queues = True
+rabbit_durable_queues = False
+{% endif -%}
+{% else -%}
+rabbit_host = {{ rabbitmq_host }}
+{% endif -%}
+{% if rabbit_ssl_port -%}
+rabbit_use_ssl = True
+rabbit_port = {{ rabbit_ssl_port }}
+{% if rabbit_ssl_ca -%}
+kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
+{% endif -%}
+{% endif -%}
+{% endif -%}

templates/parts/rabbitmq-single

@@ -0,0 +1,13 @@
+{% if rabbitmq_host -%}
+rabbit_host = {{ rabbitmq_host }}
+rabbit_userid = {{ rabbitmq_user }}
+rabbit_password = {{ rabbitmq_password }}
+rabbit_virtual_host = {{ rabbitmq_virtual_host }}
+{% if rabbit_ssl_port %}
+rabbit_use_ssl=True
+rabbit_port={{ rabbit_ssl_port }}
+{% if rabbit_ssl_ca %}
+kombu_ssl_ca_certs={{rabbit_ssl_ca}}
+{% endif -%}
+{% endif -%}
+{% endif -%}

unit_tests/test_glance_contexts.py

@@ -65,3 +65,5 @@ class TestGlanceContexts(CharmTestCase):
                                            'namespace': 'glance'})
                 self.assertTrue(mock_https.called)
                 mock_unit_get.assert_called_with('private-address')
+                self.assertTrue(mock_enable_modules.called)
+                self.assertTrue(mock_configure_cert.called)

unit_tests/test_glance_relations.py

@@ -23,6 +23,7 @@ TO_PATCH = [
     'canonical_url',
     'config',
     'juju_log',
+    'is_relation_made',
     'open_port',
     'relation_ids',
     'relation_set',
@@ -53,7 +54,6 @@ TO_PATCH = [
     'check_call',
     'execd_preinstall',
     'lsb_release',
-    'mkdir'
 ]
 
 
@@ -73,6 +73,7 @@ class GlanceRelationTests(CharmTestCase):
         self.apt_install.assert_called_with(['apache2', 'glance',
                                              'python-mysqldb',
                                              'python-swift',
+                                             'python-psycopg2',
                                              'python-keystone',
                                              'uuid', 'haproxy'])
         self.assertTrue(self.execd_preinstall.called)
@@ -88,12 +89,37 @@ class GlanceRelationTests(CharmTestCase):
 
     def test_db_joined(self):
         self.unit_get.return_value = 'glance.foohost.com'
+        self.is_relation_made.return_value = False
         relations.db_joined()
         self.relation_set.assert_called_with(database='glance',
                                              username='glance',
                                              hostname='glance.foohost.com')
         self.unit_get.assert_called_with('private-address')
 
+    def test_postgresql_db_joined(self):
+        self.unit_get.return_value = 'glance.foohost.com'
+        self.is_relation_made.return_value = False
+        relations.pgsql_db_joined()
+        self.relation_set.assert_called_with(database='glance'),
+
+    def test_db_joined_with_postgresql(self):
+        self.is_relation_made.return_value = True
+
+        with self.assertRaises(Exception) as context:
+            relations.db_joined()
+        self.assertEqual(context.exception.message,
+                         'Attempting to associate a mysql database when there '
+                         'is already associated a postgresql one')
+
+    def test_postgresql_joined_with_db(self):
+        self.is_relation_made.return_value = True
+
+        with self.assertRaises(Exception) as context:
+            relations.pgsql_db_joined()
+        self.assertEqual(context.exception.message,
+                         'Attempting to associate a postgresql database when'
+                         ' there is already associated a mysql one')
+
     @patch.object(relations, 'CONFIGS')
     def test_db_changed_missing_relation_data(self, configs):
         configs.complete_contexts = MagicMock()
@@ -103,12 +129,27 @@ class GlanceRelationTests(CharmTestCase):
             'shared-db relation incomplete. Peer not ready?'
         )
 
+    @patch.object(relations, 'CONFIGS')
+    def test_postgresql_db_changed_missing_relation_data(self, configs):
+        configs.complete_contexts = MagicMock()
+        configs.complete_contexts.return_value = []
+        relations.pgsql_db_changed()
+        self.juju_log.assert_called_with(
+            'pgsql-db relation incomplete. Peer not ready?'
+        )
+
     def _shared_db_test(self, configs):
         configs.complete_contexts = MagicMock()
         configs.complete_contexts.return_value = ['shared-db']
         configs.write = MagicMock()
         relations.db_changed()
 
+    def _postgresql_db_test(self, configs):
+        configs.complete_contexts = MagicMock()
+        configs.complete_contexts.return_value = ['pgsql-db']
+        configs.write = MagicMock()
+        relations.pgsql_db_changed()
+
     @patch.object(relations, 'CONFIGS')
     def test_db_changed_no_essex(self, configs):
         self._shared_db_test(configs)
@@ -120,6 +161,17 @@ class GlanceRelationTests(CharmTestCase):
         )
         self.migrate_database.assert_called_with()
 
+    @patch.object(relations, 'CONFIGS')
+    def test_postgresql_db_changed_no_essex(self, configs):
+        self._postgresql_db_test(configs)
+        self.assertEquals([call('/etc/glance/glance-registry.conf'),
+                           call('/etc/glance/glance-api.conf')],
+                          configs.write.call_args_list)
+        self.juju_log.assert_called_with(
+            'Cluster leader, performing db sync'
+        )
+        self.migrate_database.assert_called_with()
+
     @patch.object(relations, 'CONFIGS')
     def test_db_changed_with_essex_not_setting_version_control(self, configs):
         self.get_os_codename_package.return_value = "essex"
@@ -132,6 +184,19 @@ class GlanceRelationTests(CharmTestCase):
         )
         self.migrate_database.assert_called_with()
 
+    @patch.object(relations, 'CONFIGS')
+    def test_postgresql_db_changed_with_essex_not_setting_version_control(
+            self, configs):
+        self.get_os_codename_package.return_value = "essex"
+        self.call.return_value = 0
+        self._postgresql_db_test(configs)
+        self.assertEquals([call('/etc/glance/glance-registry.conf')],
+                          configs.write.call_args_list)
+        self.juju_log.assert_called_with(
+            'Cluster leader, performing db sync'
+        )
+        self.migrate_database.assert_called_with()
+
     @patch.object(relations, 'CONFIGS')
     def test_db_changed_with_essex_setting_version_control(self, configs):
         self.get_os_codename_package.return_value = "essex"
@@ -147,6 +212,22 @@ class GlanceRelationTests(CharmTestCase):
         )
         self.migrate_database.assert_called_with()
 
+    @patch.object(relations, 'CONFIGS')
+    def test_postgresql_db_changed_with_essex_setting_version_control(
+            self, configs):
+        self.get_os_codename_package.return_value = "essex"
+        self.call.return_value = 1
+        self._postgresql_db_test(configs)
+        self.assertEquals([call('/etc/glance/glance-registry.conf')],
+                          configs.write.call_args_list)
+        self.check_call.assert_called_with(
+            ["glance-manage", "version_control", "0"]
+        )
+        self.juju_log.assert_called_with(
+            'Cluster leader, performing db sync'
+        )
+        self.migrate_database.assert_called_with()
+
     def test_image_service_joined_not_leader(self):
         self.eligible_leader.return_value = False
         relations.image_service_joined()