Support configurable Zaqar backends

This patch adds parameters to configure alternative version
of the Zaqar messaging and management backends.

The intent is to make use of these settings in the
containers undercloud to use swift/mysql backends as a default
thus avoiding the dependency on MongoDB.

Change-Id: Ifd6a561737184c9322192ffc9a412c77d6eac3e9
Depends-On: Ie6a56b9163950cee2c0341afa0c0ddce665f3704
Depends-On: I3598e39c0a3cdf80b96e728d9aa8a7e6505e0690

docker/services/zaqar.yaml

@@ -10,6 +10,10 @@ parameters:
   DockerZaqarConfigImage:
     description: The container image to use for the zaqar config_volume
     type: string
+  ZaqarManagementStore:
+    type: string
+    description: The management store for Zaqar
+    default: mongodb
   EndpointMap:
     default: {}
     description: Mapping of service endpoint -> protocol. Typically set
@@ -37,6 +41,9 @@ parameters:
     description: Parameters specific to the role
     type: json
 
+conditions:
+  zaqar_management_store_sqlalchemy: {equals : [{get_param: ZaqarManagementStore}, 'sqlalchemy']}
+
 resources:
 
   ContainersCommon:
@@ -87,38 +94,65 @@ outputs:
               owner: zaqar:zaqar
               recurse: true
       docker_config:
-        step_4:
+        map_merge:
-          zaqar:
+          -
-            image: &zaqar_image {get_param: DockerZaqarImage}
+            if:
-            net: host
+            - zaqar_management_store_sqlalchemy
-            privileged: false
+            -
-            restart: always
+              step_2:
-            # NOTE(mandre) kolla image changes the user to 'zaqar', we need it
+                zaqar_init_log:
-            # to be root to run httpd
+                  image: &zaqar_image {get_param: DockerZaqarImage}
-            user: root
+                  user: root
-            volumes:
+                  volumes:
-              list_concat:
+                    - /var/log/containers/zaqar:/var/log/zaqar
-                - {get_attr: [ContainersCommon, volumes]}
+                  command: ['/bin/bash', '-c', 'chown -R zaqar:zaqar /var/log/zaqar']
-                -
+              step_3:
-                  - /var/lib/kolla/config_files/zaqar.json:/var/lib/kolla/config_files/config.json:ro
+                zaqar_db_sync:
-                  - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
+                  image: *zaqar_image
-                  - /var/log/containers/zaqar:/var/log/zaqar
+                  net: host
-            environment:
+                  privileged: false
-              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+                  detach: false
-          zaqar_websocket:
+                  user: root
-            image: *zaqar_image
+                  volumes:
-            net: host
+                    list_concat:
-            privileged: false
+                      - {get_attr: [ContainersCommon, volumes]}
-            restart: always
+                      -
-            volumes:
+                        - /var/lib/config-data/zaqar/etc/zaqar/:/etc/zaqar/:ro
-              list_concat:
+                        - /var/log/containers/zaqar:/var/log/zaqar
-                - {get_attr: [ContainersCommon, volumes]}
+                  command: "/usr/bin/bootstrap_host_exec zaqar su zaqar -s /bin/bash -c 'zaqar-sql-db-manage upgrade head'"
-                -
+            - {}
-                  - /var/lib/kolla/config_files/zaqar_websocket.json:/var/lib/kolla/config_files/config.json:ro
+          - step_4:
-                  - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
+              zaqar:
-                  - /var/log/containers/zaqar:/var/log/zaqar
+                image: *zaqar_image
-            environment:
+                net: host
-              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+                privileged: false
+                restart: always
+                # NOTE(mandre) kolla image changes the user to 'zaqar', we need it
+                # to be root to run httpd
+                user: root
+                volumes:
+                  list_concat:
+                    - {get_attr: [ContainersCommon, volumes]}
+                    -
+                      - /var/lib/kolla/config_files/zaqar.json:/var/lib/kolla/config_files/config.json:ro
+                      - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
+                      - /var/log/containers/zaqar:/var/log/zaqar
+                environment:
+                  - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+              zaqar_websocket:
+                image: *zaqar_image
+                net: host
+                privileged: false
+                restart: always
+                volumes:
+                  list_concat:
+                    - {get_attr: [ContainersCommon, volumes]}
+                    -
+                      - /var/lib/kolla/config_files/zaqar_websocket.json:/var/lib/kolla/config_files/config.json:ro
+                      - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
+                      - /var/log/containers/zaqar:/var/log/zaqar
+                environment:
+                  - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
         - name: create persistent logs directory
           file:

puppet/services/zaqar.yaml

@@ -56,6 +56,14 @@ parameters:
     type: string
     description: Set the number of workers for zaqar::wsgi::apache
     default: '%{::os_workers}'
+  ZaqarMessageStore:
+    type: string
+    description: The messaging store for Zaqar
+    default: mongodb
+  ZaqarManagementStore:
+    type: string
+    description: The management store for Zaqar
+    default: mongodb
   EnableInternalTLS:
     type: boolean
     default: false
@@ -63,6 +71,8 @@ parameters:
 conditions:
   zaqar_workers_zero: {equals : [{get_param: ZaqarWorkers}, 0]}
   service_debug_unset: {equals : [{get_param: ZaqarDebug}, '']}
+  zaqar_messaging_store_swift: {equals : [{get_param: ZaqarMessageStore}, 'swift']}
+  zaqar_management_store_sqlalchemy: {equals : [{get_param: ZaqarManagementStore}, 'sqlalchemy']}
 
 resources:
 
@@ -105,26 +115,67 @@ outputs:
                   "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, ZaqarApiNetwork]}
+            zaqar::message_store: {get_param: ZaqarMessageStore}
+            zaqar::management_store: {get_param: ZaqarManagementStore}
+          -
+            if:
+            - zaqar_messaging_store_swift
+            -
+              zaqar::messaging::swift::uri:
+                list_join:
+                  - ''
+                  - ['swift://zaqar:', {get_param: ZaqarPassword}, '@/service']
+              zaqar::messaging::swift::auth_url: {get_param: [EndpointMap, KeystoneInternal, uri_no_suffix] }
+              tripleo::profile::base::zaqar::messaging_store: 'swift'
+            - {}
+          -
+            if:
+            - zaqar_management_store_sqlalchemy
+            -
+              tripleo::profile::base::zaqar::management_store: 'sqlalchemy'
+              zaqar::management::sqlalchemy::uri:
+                make_url:
+                  scheme: {get_param: [EndpointMap, MysqlInternal, protocol]}
+                  username: zaqar
+                  password: {get_param: ZaqarPassword}
+                  host: {get_param: [EndpointMap, MysqlInternal, host]}
+                  path: /zaqar
+                  query:
+                    read_default_file: /etc/my.cnf.d/tripleo.cnf
+                    read_default_group: tripleo
+            - {}
           -
             if:
             - zaqar_workers_zero
             - {}
             - zaqar::wsgi::apache::workers: {get_param: ZaqarWorkers}
       service_config_settings:
-        keystone:
+        map_merge:
-          zaqar::keystone::auth::password: {get_param: ZaqarPassword}
+          - keystone:
-          zaqar::keystone::auth::public_url: {get_param: [EndpointMap, ZaqarPublic, uri]}
+              zaqar::keystone::auth::password: {get_param: ZaqarPassword}
-          zaqar::keystone::auth::admin_url: {get_param: [EndpointMap, ZaqarAdmin, uri]}
+              zaqar::keystone::auth::public_url: {get_param: [EndpointMap, ZaqarPublic, uri]}
-          zaqar::keystone::auth::internal_url: {get_param: [EndpointMap, ZaqarInternal, uri]}
+              zaqar::keystone::auth::admin_url: {get_param: [EndpointMap, ZaqarAdmin, uri]}
-          zaqar::keystone::auth::region: {get_param: KeystoneRegion}
+              zaqar::keystone::auth::internal_url: {get_param: [EndpointMap, ZaqarInternal, uri]}
-          zaqar::keystone::auth::tenant: 'service'
+              zaqar::keystone::auth::region: {get_param: KeystoneRegion}
-          zaqar::keystone::auth_websocket::password: {get_param: ZaqarPassword}
+              zaqar::keystone::auth::tenant: 'service'
-          zaqar::keystone::auth_websocket::public_url: {get_param: [EndpointMap, ZaqarWebSocketPublic, uri]}
+              zaqar::keystone::auth_websocket::password: {get_param: ZaqarPassword}
-          zaqar::keystone::auth_websocket::admin_url: {get_param: [EndpointMap, ZaqarWebSocketAdmin, uri]}
+              zaqar::keystone::auth_websocket::public_url: {get_param: [EndpointMap, ZaqarWebSocketPublic, uri]}
-          zaqar::keystone::auth_websocket::internal_url: {get_param: [EndpointMap, ZaqarWebSocketInternal, uri]}
+              zaqar::keystone::auth_websocket::admin_url: {get_param: [EndpointMap, ZaqarWebSocketAdmin, uri]}
-          zaqar::keystone::auth_websocket::region: {get_param: KeystoneRegion}
+              zaqar::keystone::auth_websocket::internal_url: {get_param: [EndpointMap, ZaqarWebSocketInternal, uri]}
-          zaqar::keystone::auth_websocket::tenant: 'service'
+              zaqar::keystone::auth_websocket::region: {get_param: KeystoneRegion}
-
+              zaqar::keystone::auth_websocket::tenant: 'service'
+          -
+            if:
+            - zaqar_management_store_sqlalchemy
+            - mysql:
+                zaqar::db::mysql::user: zaqar
+                zaqar::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
+                zaqar::db::mysql::dbname: zaqar
+                zaqar::db::mysql::password: {get_param: ZaqarPassword}
+                zaqar::db::mysql::allowed_hosts:
+                  - '%'
+                  - "%{hiera('mysql_bind_host')}"
+            - {}
       step_config: |
         include ::tripleo::profile::base::zaqar
       upgrade_tasks:

releasenotes/notes/zaqar_backends-ccf8adfd24a17bf5.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add Heat parameters which allow the end user to configure custom
+    management and messaging backends for MySQL and Swift.

tools/yaml-validate.py

@@ -219,11 +219,13 @@ def validate_docker_service(filename, tpl):
         if 'docker_config' in role_data:
             docker_config = role_data['docker_config']
             for _, step in docker_config.items():
+                if not isinstance(step, dict):
+                    # NOTE(mandre) this skips everything that is not a dict
+                    # so we may ignore some containers definitions if they
+                    # are in a map_merge for example
+                    continue
                 for _, container in step.items():
                     if not isinstance(container, dict):
-                        # NOTE(mandre) this skips everything that is not a dict
-                        # so we may ignore some containers definitions if they
-                        # are in a map_merge for example
                         continue
                     command = container.get('command', '')
                     if isinstance(command, list):