Merge "Avoid concurrent nova cell_v2 discovery instances"

container_config_scripts/nova_cell_v2_discover_hosts.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+#
+# Copyright 2018 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import logging
+import os
+import random
+import subprocess
+import sys
+import time
+
+random.seed()
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+LOG = logging.getLogger('nova_cell_v2_discover_hosts')
+
+iterations = 10
+timeout_max = 30
+nova_cfg = '/etc/nova/nova.conf'
+
+if __name__ == '__main__':
+    if not os.path.isfile(nova_cfg):
+        LOG.error('Nova configuration file %s does not exist', nova_cfg)
+        sys.exit(1)
+
+    for i in range(iterations):
+        try:
+            subprocess.check_call([
+                '/usr/bin/nova-manage',
+                'cell_v2',
+                'discover_hosts',
+                '--by-service',
+                '--verbose'
+            ])
+            sys.exit(0)
+        except subprocess.CalledProcessError as e:
+            LOG.error('Cell v2 discovery failed with exit code %d, retrying',
+                      e.returncode)
+        except Exception as e:
+            LOG.exception('Error during host discovery')
+        time.sleep(random.randint(1, timeout_max))
+sys.exit(1)
+
+# vim: set et ts=4 sw=4 :

container_config_scripts/nova_wait_for_compute_service.py

@@ -13,13 +13,10 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-from __future__ import print_function
-
 import logging
 from optparse import OptionParser
 import os
 import socket
-import subprocess
 import sys
 import time
 
@@ -32,8 +29,10 @@ from novaclient import client
 from six.moves.configparser import SafeConfigParser
 
 logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-LOG = logging.getLogger('nova_cell_v2_discover_host')
+LOG = logging.getLogger('nova_wait_for_compute_service')
 
+iterations = 60
+timeout = 10
 nova_cfg = '/etc/nova/nova.conf'
 
 if __name__ == '__main__':
@@ -77,33 +76,21 @@ if __name__ == '__main__':
     sess = session.Session(auth=auth, verify=options.insecure)
     nova = client.Client('2.11', session=sess, endpoint_type='internal')
 
-    # Wait until this host is listed in the service list then
-    # run cellv2 host discovery
-    retries = 10
-    for i in range(retries):
+    # Wait until this host is listed in the service list
+    for i in range(iterations):
         try:
             service_list = nova.services.list(binary='nova-compute')
             for entry in service_list:
                 host = getattr(entry, 'host', '')
                 zone = getattr(entry, 'zone', '')
                 if host == my_host and zone != 'internal':
-                    LOG.info('(cellv2) Service registered, running discovery')
-                    sys.exit(subprocess.call([
-                        '/usr/bin/nova-manage',
-                        'cell_v2',
-                        'discover_hosts',
-                        '--by-service',
-                        '--verbose'
-                    ]))
-                if len(service_list) == 0:
-                    LOG.warning('(cellv2) no nova-compute service registered' +
-                                ' after %i checks', i)
-            LOG.info('(cellv2) Waiting for service to register')
-        except subprocess.CalledProcessError:
-            LOG.info('(cellv2) Retrying')
+                    LOG.info('Nova-compute service registered')
+                    sys.exit(0)
+            LOG.info('Waiting for nova-compute service to register')
         except Exception as e:
-            LOG.exception('Error during host discovery:')
-        time.sleep(30)
+            LOG.exception(
+                'Error while waiting for nova-compute service to register')
+        time.sleep(timeout)
 sys.exit(1)
 
 # vim: set et ts=4 sw=4 :

deployment/containers-common.yaml

@@ -84,7 +84,7 @@ outputs:
           fi
           exit $rc
       pyshim.sh:
-        mode: "0700"
+        mode: "0755"
         content: { get_file: ../container_config_scripts/pyshim.sh }
 
   volumes_base:

deployment/nova/nova-compute-common-container-puppet.yaml

@@ -41,8 +41,11 @@ outputs:
         mode: "0700"
         content: { get_file: ../../container_config_scripts/nova_statedir_ownership.py }
       nova_wait_for_placement_service.py:
-        mode: "0700"
+        mode: "0755"
         content: { get_file: ../../container_config_scripts/nova_wait_for_placement_service.py }
-      nova_cell_v2_discover_host.py:
-        mode: "0700"
-        content: { get_file: ../../container_config_scripts/nova_cell_v2_discover_host.py }
+      nova_cell_v2_discover_hosts.py:
+        mode: "0755"
+        content: { get_file: ../../container_config_scripts/nova_cell_v2_discover_hosts.py }
+      nova_wait_for_compute_service.py:
+        mode: "0755"
+        content: { get_file: ../../container_config_scripts/nova_wait_for_compute_service.py }

deployment/nova/nova-compute-container-puppet.yaml

@@ -530,62 +530,81 @@ outputs:
                   - - 'TRIPLEO_DEPLOY_IDENTIFIER='
                     - {get_param: DeployIdentifier}
         step_4:
-          nova_wait_for_placement_service:
-            start_order: 2
-            image: *nova_compute_image
-            user: root
-            net: host
-            privileged: false
-            detach: false
-            volumes:
-              - /var/lib/container-config-scripts/:/container-config-scripts/:z
-              - /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova:ro
-            command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_wait_for_placement_service.py"
-          nova_compute:
-            start_order: 3
-            image: *nova_compute_image
-            ulimit: {get_param: DockerNovaComputeUlimit}
-            ipc: host
-            net: host
-            privileged: true
-            user: nova
-            restart: always
-            healthcheck:
-              test:
-                list_join:
-                  - ' '
-                  - - '/openstack/healthcheck'
-                    - yaql:
-                        expression: str($.data.port)
-                        data:
-                          port: {get_attr: [RabbitMQService, role_data, global_config_settings, 'oslo_messaging_rpc_port']}
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, volumes]}
-                - {get_attr: [NovaLogging, volumes]}
-                - {get_param: NovaComputeOptVolumes}
-                -
-                  - /var/lib/kolla/config_files/nova_compute.json:/var/lib/kolla/config_files/config.json:ro
-                  - /var/lib/config-data/puppet-generated/nova_libvirt/:/var/lib/kolla/config_files/src:ro
-                  - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro
-                  - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
-                  - /dev:/dev
-                  - /lib/modules:/lib/modules:ro
-                  - /run:/run
-                  - /var/lib/iscsi:/var/lib/iscsi:z
-                  - /var/lib/nova:/var/lib/nova:shared,z
-                  - /var/lib/libvirt:/var/lib/libvirt:shared,z
-                  - /sys/class/net:/sys/class/net
-                  - /sys/bus/pci:/sys/bus/pci
-            environment:
-              list_concat:
-                - {get_param: NovaComputeOptEnvVars}
-                -
-                  - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+          map_merge:
+            - nova_wait_for_placement_service:
+                start_order: 2
+                image: *nova_compute_image
+                user: nova
+                net: host
+                privileged: false
+                detach: false
+                volumes:
+                  - /var/lib/container-config-scripts/:/container-config-scripts/:z
+                  - /var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova:ro
+                command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_wait_for_placement_service.py"
+            - nova_compute:
+                start_order: 3
+                image: *nova_compute_image
+                ulimit: {get_param: DockerNovaComputeUlimit}
+                ipc: host
+                net: host
+                privileged: true
+                user: nova
+                restart: always
+                healthcheck:
+                  test:
+                    list_join:
+                      - ' '
+                      - - '/openstack/healthcheck'
+                        - yaql:
+                            expression: str($.data.port)
+                            data:
+                              port: {get_attr: [RabbitMQService, role_data, global_config_settings, 'oslo_messaging_rpc_port']}
+                volumes:
+                  list_concat:
+                    - {get_attr: [ContainersCommon, volumes]}
+                    - {get_attr: [NovaLogging, volumes]}
+                    - {get_param: NovaComputeOptVolumes}
+                    -
+                      - /var/lib/kolla/config_files/nova_compute.json:/var/lib/kolla/config_files/config.json:ro
+                      - /var/lib/config-data/puppet-generated/nova_libvirt/:/var/lib/kolla/config_files/src:ro
+                      - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro
+                      - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+                      - /dev:/dev
+                      - /lib/modules:/lib/modules:ro
+                      - /run:/run
+                      - /var/lib/iscsi:/var/lib/iscsi:z
+                      - /var/lib/nova:/var/lib/nova:shared,z
+                      - /var/lib/libvirt:/var/lib/libvirt:shared,z
+                      - /sys/class/net:/sys/class/net
+                      - /sys/bus/pci:/sys/bus/pci
+                environment:
+                  list_concat:
+                    - {get_param: NovaComputeOptEnvVars}
+                    -
+                      - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+            - if:
+                - is_not_additional_cell
+                - nova_wait_for_compute_service:
+                    start_order: 4
+                    image: *nova_compute_image
+                    net: host
+                    detach: false
+                    volumes:
+                      list_concat:
+                        - {get_attr: [ContainersCommon, volumes]}
+                        -
+                          - /var/lib/config-data/nova_libvirt/etc/my.cnf.d/:/etc/my.cnf.d/:ro
+                          - /var/lib/config-data/nova_libvirt/etc/nova/:/etc/nova/:ro
+                          - /var/log/containers/nova:/var/log/nova
+                          - /var/lib/container-config-scripts/:/container-config-scripts/
+                    user: nova
+                    command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_wait_for_compute_service.py"
+                - {}
         step_5:
           if:
             - is_not_additional_cell
-            - nova_cellv2_discover_hosts:
+            - nova_cell_v2_discover_hosts:
                 start_order: 0
                 image: *nova_compute_image
                 net: host
@@ -599,7 +618,7 @@ outputs:
                       - /var/log/containers/nova:/var/log/nova
                       - /var/lib/container-config-scripts/:/container-config-scripts/
                 user: root
-                command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_cell_v2_discover_host.py"
+                command: "/usr/bin/bootstrap_host_exec nova_compute su nova -s /bin/bash -c '/container-config-scripts/pyshim.sh /container-config-scripts/nova_cell_v2_discover_hosts.py'"
             - {}
       host_prep_tasks:
         list_concat:

deployment/nova/nova-ironic-container-puppet.yaml

@@ -127,8 +127,9 @@ outputs:
               - /var/lib/nova:/var/lib/nova:shared,z
               - /var/lib/container-config-scripts/:/container-config-scripts/
             command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_statedir_ownership.py"
-        step_5:
+        step_4:
           nova_compute:
+            start_order: 100 # After the ironic services
             image: *nova_ironic_image
             net: host
             privileged: true
@@ -157,8 +158,8 @@ outputs:
                   - /var/log/containers/nova:/var/log/nova:z
             environment:
              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-          nova_cellv2_discover_hosts:
-            start_order: 1
+          nova_wait_for_compute_service:
+            start_order: 101
             image: *nova_ironic_image
             net: host
             detach: false
@@ -171,7 +172,23 @@ outputs:
                   - /var/log/containers/nova:/var/log/nova
                   - /var/lib/container-config-scripts/:/container-config-scripts/
             user: root
-            command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_cell_v2_discover_host.py"
+            command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_wait_for_compute_service.py"
+        step_5:
+          nova_cell_v2_discover_hosts:
+            start_order: 0
+            image: *nova_ironic_image
+            net: host
+            detach: false
+            volumes:
+              list_concat:
+                - {get_attr: [ContainersCommon, volumes]}
+                -
+                  - /var/lib/config-data/nova/etc/my.cnf.d/:/etc/my.cnf.d/:ro
+                  - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
+                  - /var/log/containers/nova:/var/log/nova
+                  - /var/lib/container-config-scripts/:/container-config-scripts/
+            user: root
+            command: "/usr/bin/bootstrap_host_exec nova_ironic su nova -s /bin/bash -c '/container-config-scripts/pyshim.sh /container-config-scripts/nova_cell_v2_discover_hosts.py'"
       host_prep_tasks:
         - name: create persistent directories
           file: