cell_v2 multi-cell

- uses split-control-plane
- adds a new CellController role
  - nova-conductor, message rpc (not notifications) and db
- move nova dbsync from nova-api to nova-conductor
  - nova db is more tightly coupled to conductor/computes
  - we don't have a nova-api services on a CellController
  - super-conductor on Controller will sync cell0 db
- new 'magic' MysqlCellInternal endpoint
  - always refers the to local MysqlInternal endpoint
  - identical to MysqlInternal for regular deployment
  - but doesn't get overridden when inheriting EndpointMap from parent
    control-plane stack
- duplicate service node name hiera for transport_urls on cell stack
  - nova -> cell oslo messaging rpc nodes
  - neutron agent -> global messaging rpc nodes
- run cell host discovery only on default cell, for additional cells
the cell needs to be created first

bp tripleo-multicell-basic

Co-Authored-By: Martin Schuppert <mschuppert@redhat.com>

Change-Id: Ife9bf12d3a6011906fa8d9f97f7524b51aef906a
Depends-On: I79c1080605611c5c7748a28d2afcc9c7275a2e5d

deployment/nova/nova-api-container-puppet.yaml

@@ -260,14 +260,7 @@ outputs:
         mysql:
           map_merge:
           - {get_attr: [NovaBase, role_data, service_config_settings, mysql]}
-          - nova::db::mysql::password: {get_param: NovaPassword}
+          - nova::db::mysql_api::password: {get_param: NovaPassword}
-            nova::db::mysql::user: nova
-            nova::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
-            nova::db::mysql::dbname: nova
-            nova::db::mysql::allowed_hosts:
-              - '%'
-              - "%{hiera('mysql_bind_host')}"
-            nova::db::mysql_api::password: {get_param: NovaPassword}
             nova::db::mysql_api::user: nova_api
             nova::db::mysql_api::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
             nova::db::mysql_api::dbname: nova_api
@@ -372,7 +365,7 @@ outputs:
           get_attr: [NovaApiLogging, docker_config, step_2]
         step_3:
           nova_api_db_sync:
-            start_order: 0
+            start_order: 0 # Runs before nova-conductor dbsync
             image: &nova_api_image {get_param: DockerNovaApiImage}
             net: host
             detach: false
@@ -386,7 +379,7 @@ outputs:
                   - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
             command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage api_db sync'"
           nova_api_map_cell0:
-            start_order: 1
+            start_order: 1 # Runs before nova-conductor dbsync
             image: *nova_api_image
             net: host
             detach: false
@@ -415,7 +408,7 @@ outputs:
                       - '?'
                       - '{query}'
           nova_api_ensure_default_cell:
-            start_order: 2
+            start_order: 2 # Runs before nova-conductor dbsync
             image: *nova_api_image
             net: host
             detach: false
@@ -427,14 +420,6 @@ outputs:
                   - /var/lib/docker-config-scripts/nova_api_ensure_default_cell.sh:/nova_api_ensure_default_cell.sh:ro
             user: root
             command: "/usr/bin/bootstrap_host_exec nova_api /nova_api_ensure_default_cell.sh"
-          nova_db_sync:
-            start_order: 3
-            image: *nova_api_image
-            net: host
-            detach: false
-            volumes: *nova_api_bootstrap_volumes
-            user: root
-            command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage db sync'"
         step_4:
           nova_api:
             start_order: 2

deployment/nova/nova-compute-container-puppet.yaml

@@ -248,6 +248,10 @@ parameters:
     description: >
       Setting this to a unique value will re-run any deployment tasks which
       perform configuration on a Heat stack-update.
+  NovaAdditionalCell:
+    default: false
+    description: Whether this is an cell additional to the default cell.
+    type: boolean
 
 resources:
 
@@ -331,6 +335,8 @@ conditions:
       -  equals: [{get_param: NovaLibvirtFileBackedMemory}, '']
       -  equals: [{get_param: NovaLibvirtFileBackedMemory}, 0]
 
+  is_not_additional_cell: {equals: [{get_param: NovaAdditionalCell}, false]}
+
 outputs:
   role_data:
     description: Role data for the Nova Compute service.
@@ -550,21 +556,23 @@ outputs:
                 -
                   - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
         step_5:
-          nova_cellv2_discover_hosts:
+          if:
-            start_order: 0
+            - is_not_additional_cell
-            image: *nova_compute_image
+            - nova_cellv2_discover_hosts:
-            net: host
+                start_order: 0
-            detach: false
+                image: *nova_compute_image
-            volumes:
+                net: host
-              list_concat:
+                detach: false
-                - {get_attr: [ContainersCommon, volumes]}
+                volumes:
-                -
+                  list_concat:
-                  - /var/lib/config-data/nova_libvirt/etc/my.cnf.d/:/etc/my.cnf.d/:ro
+                    - {get_attr: [ContainersCommon, volumes]}
-                  - /var/lib/config-data/nova_libvirt/etc/nova/:/etc/nova/:ro
+                    -
-                  - /var/log/containers/nova:/var/log/nova
+                      - /var/lib/config-data/nova_libvirt/etc/nova/:/etc/nova/:ro
-                  - /var/lib/docker-config-scripts/:/docker-config-scripts/
+                      - /var/log/containers/nova:/var/log/nova
-            user: root
+                      - /var/lib/docker-config-scripts/:/docker-config-scripts/
-            command: "/docker-config-scripts/pyshim.sh /docker-config-scripts/nova_cell_v2_discover_host.py"
+                user: root
+                command: "/docker-config-scripts/pyshim.sh /docker-config-scripts/nova_cell_v2_discover_host.py"
+            - {}
       host_prep_tasks:
         list_concat:
         - {get_attr: [NovaLogging, host_prep_tasks]}

deployment/nova/nova-conductor-container-puppet.yaml

@@ -56,6 +56,10 @@ parameters:
     type: string
     description: Nova Compute upgrade level
     default: ''
+  NovaPassword:
+    description: The password for the nova service and db account
+    type: string
+    hidden: true
 
 conditions:
   nova_workers_zero: {equals : [{get_param: NovaWorkers}, 0]}
@@ -104,6 +108,16 @@ outputs:
             - nova
           tripleo_fluentd_sources_nova_conductor:
             - {get_param: NovaConductorLoggingSource}
+        mysql:
+          map_merge:
+          - {get_attr: [NovaBase, role_data, service_config_settings, mysql]}
+          - nova::db::mysql::password: {get_param: NovaPassword}
+            nova::db::mysql::user: nova
+            nova::db::mysql::host: {get_param: [EndpointMap, MysqlCellInternal, host_nobrackets]}
+            nova::db::mysql::dbname: nova
+            nova::db::mysql::allowed_hosts:
+              - '%'
+              - "%{hiera('mysql_bind_host')}"
       # BEGIN DOCKER SETTINGS
       puppet_config:
         config_volume: nova
@@ -131,21 +145,31 @@ outputs:
               owner: nova:nova
               recurse: true
       docker_config:
+        step_2:
+          get_attr: [NovaLogging, docker_config, step_2]
+        step_3:
+          nova_db_sync:
+            image: &nova_conductor_image {get_param: DockerNovaConductorImage}
+            start_order: 3 # Runs after nova-api tasks if installed on this host
+            net: host
+            detach: false
+            volumes: &nova_conductor_bootstrap_volumes
+              list_concat:
+                - {get_attr: [ContainersCommon, volumes]}
+                - {get_attr: [NovaLogging, volumes]}
+                -
+                  - /var/lib/config-data/nova/etc/my.cnf.d/tripleo.cnf:/etc/my.cnf.d/tripleo.cnf:ro
+                  - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
+            user: root
+            command: "/usr/bin/bootstrap_host_exec nova_conductor su nova -s /bin/bash -c '/usr/bin/nova-manage db sync'"
         step_4:
           nova_conductor:
-            image: {get_param: DockerNovaConductorImage}
+            image: *nova_conductor_image
             net: host
             privileged: false
             restart: always
             healthcheck:
-              test:
+              test: /openstack/healthcheck
-                list_join:
-                  - ' '
-                  - - '/openstack/healthcheck'
-                    - yaql:
-                        expression: str($.data.port)
-                        data:
-                          port: {get_attr: [NovaBase, role_data, config_settings, 'nova::rabbit_port']}
             volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}

deployment/nova/nova-scheduler-container-puppet.yaml

@@ -117,8 +117,7 @@ outputs:
         map_merge:
           - {get_attr: [NovaBase, role_data, config_settings]}
           - {get_attr: [NovaLogging, config_settings]}
-          - nova::ram_allocation_ratio: '1.0'
+          - nova::scheduler::filter::scheduler_available_filters: {get_param: NovaSchedulerAvailableFilters}
-            nova::scheduler::filter::scheduler_available_filters: {get_param: NovaSchedulerAvailableFilters}
             nova::scheduler::filter::scheduler_default_filters: {get_param: NovaSchedulerDefaultFilters}
             nova::scheduler::filter::scheduler_max_attempts: {get_param: NovaSchedulerMaxAttempts}
             nova::scheduler::discover_hosts_in_cells_interval: {get_param: NovaSchedulerDiscoverHostsInCellsInterval}

network/ports/net_ip_list_map.j2.yaml

@@ -25,7 +25,13 @@ parameters:
   NetworkHostnameMap:
     default: []
     type: json
+  NovaAdditionalCell:
+    default: false
+    description: Whether this is an cell additional to the default cell.
+    type: boolean
 
+conditions:
+  is_cell: {equals: [{get_param: NovaAdditionalCell}, true]}
 
 resources:
    # This adds the extra "services" on for keystone
@@ -123,11 +129,20 @@ outputs:
               map:
                 map_replace:
                   - map_merge:
-                      repeat:
+                      if:
-                        template:
+                        - is_cell
-                          SERVICE_node_names: SERVICE_network
+                        -
-                        for_each:
+                          repeat:
-                          SERVICE: {get_attr: [EnabledServicesValue, value]}
+                            template:
+                              SERVICE_cell_node_names: SERVICE_network
+                            for_each:
+                              SERVICE: {get_attr: [EnabledServicesValue, value]}
+                        -
+                          repeat:
+                            template:
+                              SERVICE_node_names: SERVICE_network
+                            for_each:
+                              SERVICE: {get_attr: [EnabledServicesValue, value]}
                   - values: {get_param: ServiceNetMap}
         - values: {get_param: NetworkHostnameMap}
   short_service_hostnames:

overcloud.j2.yaml

@@ -390,6 +390,7 @@ resources:
         map_merge:
           - {get_attr: [EndpointMap, endpoint_map]}
           - {get_param: EndpointMapOverride}
+          - MysqlCellInternal: {get_attr: [EndpointMap, endpoint_map, MysqlInternal]}
 
   SshKnownHostsConfig:
     type: OS::TripleO::Ssh::KnownHostsConfig

puppet/services/nova-base.yaml

@@ -217,6 +217,10 @@ parameters:
         Messaging client subscriber parameter to specify
         an SSL connection to the messaging host.
     type: string
+  NovaAdditionalCell:
+    default: false
+    description: Whether this is an cell additional to the default cell.
+    type: boolean
 
 conditions:
 
@@ -262,10 +266,10 @@ outputs:
           nova::placement::valid_interfaces: {get_param: NovaPlacementAPIInterface}
           nova::database_connection:
             make_url:
-              scheme: {get_param: [EndpointMap, MysqlInternal, protocol]}
+              scheme: {get_param: [EndpointMap, MysqlCellInternal, protocol]}
               username: nova
               password: {get_param: NovaPassword}
-              host: {get_param: [EndpointMap, MysqlInternal, host]}
+              host: {get_param: [EndpointMap, MysqlCellInternal, host]}
               path: /nova
               query:
                 read_default_file: /etc/my.cnf.d/tripleo.cnf
@@ -348,6 +352,8 @@ outputs:
           nova::cron::purge_shadow_tables::verbose: {get_param: NovaCronPurgeShadowTablesVerbose}
           nova::cron::purge_shadow_tables::all_cells: {get_param: NovaCronPurgeShadowTablesAllCells}
           nova::compute::sync_power_state_interval: {get_param: NovaSyncPowerStateInterval}
+          nova::ram_allocation_ratio: '1.0'
+          nova_is_additional_cell: {get_param: NovaAdditionalCell}
         - get_attr: [RoleParametersValue, value]
         -
           if:

roles/CellController.yaml

@@ -0,0 +1,56 @@
+###############################################################################
+# Role: CellController                                                        #
+###############################################################################
+- name: CellController
+  description: |
+    CellController role for the nova cell_v2 controler services
+  CountDefault: 1
+  tags:
+    - primary
+    - controller
+  networks:
+    - External
+    - InternalApi
+    - Storage
+    - StorageMgmt
+    - Tenant
+  # For systems with both IPv4 and IPv6, you may specify a gateway network for
+  # each, such as ['ControlPlane', 'External']
+  default_route_networks: ['External']
+  HostnameFormatDefault: '%stackname%-cellcontrol-%index%'
+  ServicesDefault:
+    - OS::TripleO::Services::Aide
+    - OS::TripleO::Services::AuditD
+    - OS::TripleO::Services::CACerts
+    - OS::TripleO::Services::CertmongerUser
+    - OS::TripleO::Services::Clustercheck
+    - OS::TripleO::Services::Collectd
+    - OS::TripleO::Services::Docker
+    - OS::TripleO::Services::Fluentd
+    - OS::TripleO::Services::HAproxy
+    - OS::TripleO::Services::Ipsec
+    - OS::TripleO::Services::Iscsid
+    - OS::TripleO::Services::Keepalived
+    - OS::TripleO::Services::Kernel
+    - OS::TripleO::Services::LoginDefs
+    - OS::TripleO::Services::MetricsQdr
+    - OS::TripleO::Services::MySQL
+    - OS::TripleO::Services::MySQLClient
+    - OS::TripleO::Services::NovaConductor
+    - OS::TripleO::Services::Ntp
+    - OS::TripleO::Services::ContainersLogrotateCrond
+    - OS::TripleO::Services::Pacemaker
+    - OS::TripleO::Services::OsloMessagingRpc
+    - OS::TripleO::Services::Rhsm
+    - OS::TripleO::Services::RsyslogSidecar
+    - OS::TripleO::Services::Securetty
+    - OS::TripleO::Services::SensuClient
+    - OS::TripleO::Services::SkydiveAgent
+    - OS::TripleO::Services::Snmp
+    - OS::TripleO::Services::Sshd
+    - OS::TripleO::Services::Timezone
+    - OS::TripleO::Services::TripleoFirewall
+    - OS::TripleO::Services::TripleoPackages
+    - OS::TripleO::Services::Tuned
+    - OS::TripleO::Services::Vpp
+    - OS::TripleO::Services::Ptp