Merge "Expose --ansible-forks"

releasenotes/notes/ansible-forks-arg-9f7b439e4b6980dd.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    A new `--ansible-forks` argument has been added to the TripleO and Overcloud
+    commands. The default value for forks has also been adjusted to no longer
+    exceed 100 forks.

tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py

@@ -1540,7 +1540,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                        deployment_options={},
                        deployment_timeout=448,  # 451 - 3, total time left
                        in_flight_validations=False, limit_hosts=None,
-                       skip_tags=None, tags=None, timeout=42, verbosity=3)],
+                       skip_tags=None, tags=None, timeout=42, verbosity=3,
+                       forks=None)],
             fixture.mock_config_download.mock_calls)
         fixture.mock_config_download.assert_called()
         mock_copy.assert_called_once()
@@ -1593,7 +1594,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                 reproduce_command=True, skip_tags='opendev-validation',
                 ssh_user='tripleo-admin', tags=None,
                 timeout=240,
-                verbosity=3, workdir=mock.ANY)],
+                verbosity=3, workdir=mock.ANY, forks=None)],
             utils_fixture2.mock_run_ansible_playbook.mock_calls)
 
     def test_download_missing_files_from_plan(self):

tripleoclient/tests/v1/overcloud_node/test_overcloud_node.py

@@ -261,7 +261,8 @@ class TestDeleteNode(fakes.TestDeleteNode):
                 extra_env_variables={'ANSIBLE_BECOME': True},
                 extra_vars=None,
                 tags=None,
-                timeout=90
+                timeout=90,
+                forks=None
             ),
             mock.call(
                 inventory='localhost,',

tripleoclient/utils.py

@@ -247,7 +247,7 @@ def run_ansible_playbook(playbook, inventory, workdir, playbook_dir=None,
                          callback_whitelist=constants.ANSIBLE_CWL,
                          ansible_cfg=None, ansible_timeout=30,
                          reproduce_command=False,
-                         timeout=None):
+                         timeout=None, forks=None):
     """Simple wrapper for ansible-playbook.
 
     :param playbook: Playbook filename.
@@ -450,6 +450,9 @@ def run_ansible_playbook(playbook, inventory, workdir, playbook_dir=None,
     if output_callback not in callback_whitelist.split(','):
         callback_whitelist = ','.join([callback_whitelist, output_callback])
 
+    if not forks:
+        forks = min(multiprocessing.cpu_count() * 4, 100)
+
     env = dict()
     env['ANSIBLE_SSH_ARGS'] = (
         '-o UserKnownHostsFile={} '
@@ -467,7 +470,7 @@ def run_ansible_playbook(playbook, inventory, workdir, playbook_dir=None,
         '-T'
     ).format(os.devnull)
     env['ANSIBLE_DISPLAY_FAILED_STDERR'] = True
-    env['ANSIBLE_FORKS'] = multiprocessing.cpu_count() * 4
+    env['ANSIBLE_FORKS'] = forks
     env['ANSIBLE_TIMEOUT'] = ansible_timeout
     env['ANSIBLE_GATHER_TIMEOUT'] = 45
     env['ANSIBLE_SSH_RETRIES'] = 3

tripleoclient/v1/overcloud_deploy.py

@@ -1023,6 +1023,14 @@ class DeployOvercloud(command.Command):
             help=_('A list of tags to skip when running the the'
                    ' config-download ansible-playbook command.')
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
         return parser
 
     def take_action(self, parsed_args):
@@ -1137,7 +1145,8 @@ class DeployOvercloud(command.Command):
                     skip_tags=parsed_args.skip_tags,
                     limit_hosts=utils.playbook_limit_parse(
                         limit_nodes=parsed_args.limit
-                    )
+                    ),
+                    forks=parsed_args.ansible_forks
                 )
                 deployment.set_deployment_status(
                     clients=self.clients,

tripleoclient/v1/overcloud_external_update.py

@@ -106,6 +106,14 @@ class ExternalUpdateRun(command.Command):
                    "execution will be limited to. For example: --limit"
                    " \"compute-0,compute-1,compute-5\".")
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
 
         return parser
 
@@ -148,6 +156,7 @@ class ExternalUpdateRun(command.Command):
             skip_tags=parsed_args.skip_tags,
             limit_hosts=oooutils.playbook_limit_parse(
                 limit_nodes=parsed_args.limit
-            )
+            ),
+            forks=parsed_args.ansible_forks
         )
         self.log.info("Completed Overcloud External Update Run.")

tripleoclient/v1/overcloud_external_upgrade.py

@@ -106,6 +106,14 @@ class ExternalUpgradeRun(command.Command):
                    "execution will be limited to. For example: --limit"
                    " \"compute-0,compute-1,compute-5\".")
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
 
         return parser
 
@@ -144,6 +152,7 @@ class ExternalUpgradeRun(command.Command):
             tags=parsed_args.tags,
             skip_tags=parsed_args.skip_tags,
             limit_hosts=oooutils.playbook_limit_parse(
-                limit_nodes=parsed_args.limit)
+                limit_nodes=parsed_args.limit),
+            forks=parsed_args.ansible_forks
         )
         self.log.info("Completed Overcloud External Upgrade Run.")

tripleoclient/v1/overcloud_update.py

@@ -169,6 +169,14 @@ class UpdateRun(command.Command):
             help=_("Use -y or --yes to skip the confirmation required before "
                    "any update operation. Use this with caution! "),
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
         return parser
 
     def take_action(self, parsed_args):
@@ -214,7 +222,8 @@ class UpdateRun(command.Command):
                 limit_nodes=parsed_args.limit
             ),
             skip_tags=parsed_args.skip_tags,
-            tags=parsed_args.tags
+            tags=parsed_args.tags,
+            forks=parsed_args.ansible_forks
         )
         self.log.info("Completed Overcloud Minor Update Run.")
 

tripleoclient/v1/overcloud_upgrade.py

@@ -204,6 +204,14 @@ class UpgradeRun(command.Command):
                                    "required before any upgrade "
                                    "operation. Use this with caution! ")
                             )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
         return parser
 
     def take_action(self, parsed_args):
@@ -250,7 +258,8 @@ class UpgradeRun(command.Command):
             skip_tags=parsed_args.skip_tags,
             limit_hosts=oooutils.playbook_limit_parse(
                 limit_nodes=parsed_args.limit
-            )
+            ),
+            forks=parsed_args.ansible_forks
         )
         self.log.info("Completed Overcloud Major Upgrade Run.")
 

tripleoclient/v1/tripleo_deploy.py

@@ -1106,6 +1106,14 @@ class Deploy(command.Command):
                    'Use "local" for standalone deployments. '
                    'Defaults to "local".')
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
 
         stack_action_group = parser.add_mutually_exclusive_group()
 
@@ -1348,6 +1356,7 @@ class Deploy(command.Command):
                             workdir=self.ansible_dir,
                             verbosity=utils.playbook_verbosity(self=self),
                             extra_env_variables=extra_env_var,
+                            forks=parsed_args.ansible_forks,
                             **operation)
             is_complete = True
         finally:

tripleoclient/workflows/deployment.py

@@ -275,7 +275,7 @@ def config_download(log, clients, stack, ssh_network='ctlplane',
                     ansible_playbook_name='deploy_steps_playbook.yaml',
                     limit_hosts=None, extra_vars=None, inventory_path=None,
                     ssh_user='tripleo-admin', tags=None, skip_tags=None,
-                    deployment_timeout=None):
+                    deployment_timeout=None, forks=None):
     """Run config download.
 
     :param log: Logging object
@@ -460,6 +460,7 @@ def config_download(log, clients, stack, ssh_network='ctlplane',
             },
             extra_vars=extra_vars,
             timeout=deployment_timeout,
+            forks=forks
         )
 
     _log_and_print(