Move nova cell v2 discovery to deploy_steps_tasks

Recent changes for e.g edge scenarios caused intended move of discovery
from controller to bootstrap compute node. The task is triggered by
deploy-identifier to make sure it gets run on any deploy,scale, ... run.
If deploy run is triggered with --skip-deploy-identifier flag, discovery
will not be triggered at and as result causing failures in previously
supported scenarios.
This change moves the host discovery task to be an ansible
deploy_steps_tasks that it gets triggered even if --skip-deploy-identifier
is used, or the compute bootstrap node is blacklisted.

Closes-Bug: #1831711

 Conflicts:
	deployment/nova/nova-compute-container-puppet.yaml
	docker/services/nova-compute-common.yaml
	docker/services/nova-ironic.yaml
	docker_config_scripts/nova_cell_v2_discover_hosts.py

Change-Id: I4bd8489e4f79e3e1bfe9338ed3043241dd605dcb
(cherry picked from commit f8779e5023)
(cherry picked from commit b82417126b)
(cherry picked from commit 58b65c1da8)

docker/services/nova-compute-common.yaml

@@ -43,9 +43,30 @@ outputs:
       nova_wait_for_placement_service.py:
         mode: "0755"
         content: { get_file: ../../docker_config_scripts/nova_wait_for_placement_service.py }
-      nova_cell_v2_discover_hosts.py:
-        mode: "0755"
-        content: { get_file: ../../docker_config_scripts/nova_cell_v2_discover_hosts.py }
       nova_wait_for_compute_service.py:
         mode: "0755"
         content: { get_file: ../../docker_config_scripts/nova_wait_for_compute_service.py }
+
+  nova_compute_common_deploy_steps_tasks:
+    description: Common host prep tasks for nova-compute services (compute + ironic)
+    value: &nova_compute_common_deploy_steps_tasks
+      - when: step|int == 5
+        block:
+          - name: discover via nova_compute?
+            set_fact:
+              delegate_host: "{{ groups['nova_compute'][0] }}"
+            when:
+              - groups['nova_compute'] is defined and (groups['nova_compute']|length>0)
+          - name: discover via nova_ironic?
+            set_fact:
+              delegate_host: "{{ groups['nova_ironic'][0] }}"
+            when:
+              - delegate_host is not defined
+              - groups['nova_ironic'] is defined and (groups['nova_ironic']|length>0)
+          - name: Discovering nova hosts
+            command: "docker exec nova_compute nova-manage cell_v2 discover_hosts --by-service"
+            become: true
+            changed_when: False
+            when:
+            - delegate_host is defined
+            - inventory_hostname == delegate_host

docker/services/nova-compute.yaml

@@ -265,29 +265,7 @@ outputs:
                       - /var/lib/docker-config-scripts/:/docker-config-scripts/
                 user: nova
                 command: "/docker-config-scripts/nova_wait_for_compute_service.py"
-        step_5:
-          nova_cell_v2_discover_hosts:
-            start_order: 0
-            image: *nova_compute_image
-            net: host
-            detach: false
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, volumes]}
-                -
-                  - /var/lib/config-data/nova_libvirt/etc/my.cnf.d/:/etc/my.cnf.d/:ro
-                  - /var/lib/config-data/nova_libvirt/etc/nova/:/etc/nova/:ro
-                  - /var/log/containers/nova:/var/log/nova
-                  - /var/lib/docker-config-scripts/:/docker-config-scripts/
-            user: root
-            command: "/usr/bin/bootstrap_host_exec nova_compute su nova -s /bin/bash -c '/docker-config-scripts/nova_cell_v2_discover_hosts.py'"
-            environment:
-              # NOTE: this should force this container to re-run on each
-              # update (scale-out, etc.)
-              - list_join:
-                  - ''
-                  - - 'TRIPLEO_DEPLOY_IDENTIFIER='
-                    - {get_param: DeployIdentifier}
+      deploy_steps_tasks: {get_attr: [NovaComputeCommon, nova_compute_common_deploy_steps_tasks]}
       host_prep_tasks:
         list_concat:
         - {get_attr: [NovaLogging, host_prep_tasks]}

docker/services/nova-ironic.yaml

@@ -160,22 +160,7 @@ outputs:
                   - /var/lib/docker-config-scripts/:/docker-config-scripts/
             user: root
             command: "/docker-config-scripts/nova_wait_for_compute_service.py"
-        step_5:
-          nova_cell_v2_discover_hosts:
-            start_order: 0
-            image: *nova_ironic_image
-            net: host
-            detach: false
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, volumes]}
-                -
-                  - /var/lib/config-data/nova/etc/my.cnf.d/:/etc/my.cnf.d/:ro
-                  - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
-                  - /var/log/containers/nova:/var/log/nova
-                  - /var/lib/docker-config-scripts/:/docker-config-scripts/
-            user: root
-            command: "su nova -s /bin/bash -c '/docker-config-scripts/nova_cell_v2_discover_hosts.py'"
+      deploy_steps_tasks: {get_attr: [NovaComputeCommon, nova_compute_common_deploy_steps_tasks]}
       host_prep_tasks:
         - name: create persistent directories
           file:

docker_config_scripts/nova_cell_v2_discover_hosts.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2018 Red Hat Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-import logging
-import os
-import random
-import subprocess
-import sys
-import time
-
-random.seed()
-
-logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
-LOG = logging.getLogger('nova_cell_v2_discover_hosts')
-
-iterations = 10
-timeout_max = 30
-nova_cfg = '/etc/nova/nova.conf'
-
-if __name__ == '__main__':
-    if not os.path.isfile(nova_cfg):
-        LOG.error('Nova configuration file %s does not exist', nova_cfg)
-        sys.exit(1)
-
-    for i in range(iterations):
-        try:
-            subprocess.check_call([
-                '/usr/bin/nova-manage',
-                'cell_v2',
-                'discover_hosts',
-                '--by-service',
-                '--verbose'
-            ])
-            sys.exit(0)
-        except subprocess.CalledProcessError as e:
-            LOG.error('Cell v2 discovery failed with exit code %d, retrying',
-                      e.returncode)
-        except Exception as e:
-            LOG.exception('Error during host discovery')
-        time.sleep(random.randint(1, timeout_max))
-sys.exit(1)
-
-# vim: set et ts=4 sw=4 :

releasenotes/notes/nova_move_host_discovery_to_external_post_deploy_tasks-e978560ee59b8b56.yaml

@@ -0,0 +1,12 @@
+---
+fixes:
+  - |
+    Recent changes for e.g edge scenarios caused intended move of discovery
+    from controller to bootstrap compute node. The task is triggered by
+    deploy-identifier to make sure it gets run on any deploy,scale, ... run.
+    If deploy run is triggered with --skip-deploy-identifier flag, discovery
+    will not be triggered at and as result causing failures in previously
+    supported scenarios.
+    This change moves the host discovery task to be an ansible
+    deploy_steps_tasks that it gets triggered even if --skip-deploy-identifier
+    is used, or the compute bootstrap node is blacklisted.

tools/yaml-validate.py

@@ -42,6 +42,7 @@ OPTIONAL_SECTIONS = ['workflow_tasks', 'cellv2_discovery']
 REQUIRED_DOCKER_SECTIONS = ['service_name', 'docker_config', 'puppet_config',
                             'config_settings']
 OPTIONAL_DOCKER_SECTIONS = ['docker_puppet_tasks', 'upgrade_tasks',
+                            'deploy_steps_tasks',
                             'pre_upgrade_rolling_tasks',
                             'fast_forward_upgrade_tasks',
                             'fast_forward_post_upgrade_tasks',