Expose --ansible-forks

Ansible forks are now configurable for a deployment. The default
will also no longer exceed 100 forks (but --ansible-forks can).

Conflicts:
    tripleoclient/v1/tripleo_deploy.py

Change-Id: Iaa6763a3124e45c1f0297cd59cc57f206dfc5cda
Depends-On: I57345d5b100efce143fa940b56c81f5e6bc6c390
Signed-off-by: Luke Short <ekultails@gmail.com>
(cherry picked from commit 896a397b1e)

releasenotes/notes/ansible-forks-arg-9f7b439e4b6980dd.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    A new `--ansible-forks` argument has been added to the TripleO and Overcloud
+    commands. The default value for forks has also been adjusted to no longer
+    exceed 100 forks.

tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py

@@ -1526,7 +1526,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                        deployment_options={},
                        deployment_timeout=448,  # 451 - 3, total time left
                        in_flight_validations=False, limit_hosts=None,
-                       skip_tags=None, tags=None, timeout=42, verbosity=3)],
+                       skip_tags=None, tags=None, timeout=42, verbosity=3,
+                       forks=None)],
             fixture.mock_config_download.mock_calls)
         fixture.mock_config_download.assert_called()
         mock_copy.assert_called_once()
@@ -1579,7 +1580,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                 reproduce_command=True, skip_tags='opendev-validation',
                 ssh_user='tripleo-admin', tags=None,
                 timeout=240,
-                verbosity=3, workdir=mock.ANY)],
+                verbosity=3, workdir=mock.ANY, forks=None)],
             utils_fixture2.mock_run_ansible_playbook.mock_calls)
 
     def test_download_missing_files_from_plan(self):

tripleoclient/tests/v1/overcloud_node/test_overcloud_node.py

@@ -257,7 +257,8 @@ class TestDeleteNode(fakes.TestDeleteNode):
                 extra_env_variables={'ANSIBLE_BECOME': True},
                 extra_vars=None,
                 tags=None,
-                timeout=90
+                timeout=90,
+                forks=None
             ),
             mock.call(
                 inventory='localhost,',

tripleoclient/utils.py

@@ -251,7 +251,7 @@ def run_ansible_playbook(playbook, inventory, workdir, playbook_dir=None,
                          callback_whitelist=constants.ANSIBLE_CWL,
                          ansible_cfg=None, ansible_timeout=30,
                          reproduce_command=False,
-                         timeout=None):
+                         timeout=None, forks=None):
     """Simple wrapper for ansible-playbook.
 
     :param playbook: Playbook filename.
@@ -453,6 +453,9 @@ def run_ansible_playbook(playbook, inventory, workdir, playbook_dir=None,
     if output_callback not in callback_whitelist.split(','):
         callback_whitelist = ','.join([callback_whitelist, output_callback])
 
+    if not forks:
+        forks = min(multiprocessing.cpu_count() * 4, 100)
+
     env = dict()
     env['ANSIBLE_SSH_ARGS'] = (
         '-o UserKnownHostsFile={} '
@@ -470,7 +473,7 @@ def run_ansible_playbook(playbook, inventory, workdir, playbook_dir=None,
         '-T'
     ).format(os.devnull)
     env['ANSIBLE_DISPLAY_FAILED_STDERR'] = True
-    env['ANSIBLE_FORKS'] = multiprocessing.cpu_count() * 4
+    env['ANSIBLE_FORKS'] = forks
     env['ANSIBLE_TIMEOUT'] = ansible_timeout
     env['ANSIBLE_GATHER_TIMEOUT'] = 45
     env['ANSIBLE_SSH_RETRIES'] = 3

tripleoclient/v1/overcloud_deploy.py

@@ -989,6 +989,14 @@ class DeployOvercloud(command.Command):
             help=_('A list of tags to skip when running the the'
                    ' config-download ansible-playbook command.')
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
         return parser
 
     def take_action(self, parsed_args):
@@ -1103,7 +1111,8 @@ class DeployOvercloud(command.Command):
                     skip_tags=parsed_args.skip_tags,
                     limit_hosts=utils.playbook_limit_parse(
                         limit_nodes=parsed_args.limit
-                    )
+                    ),
+                    forks=parsed_args.ansible_forks
                 )
                 deployment.set_deployment_status(
                     clients=self.clients,

tripleoclient/v1/overcloud_external_update.py

@@ -108,6 +108,14 @@ class ExternalUpdateRun(command.Command):
                    "execution will be limited to. For example: --limit"
                    " \"compute-0,compute-1,compute-5\".")
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
 
         return parser
 
@@ -147,6 +155,7 @@ class ExternalUpdateRun(command.Command):
             skip_tags=parsed_args.skip_tags,
             limit_hosts=oooutils.playbook_limit_parse(
                 limit_nodes=parsed_args.limit
-            )
+            ),
+            forks=parsed_args.ansible_forks
         )
         self.log.info("Completed Overcloud External Update Run.")

tripleoclient/v1/overcloud_external_upgrade.py

@@ -108,6 +108,14 @@ class ExternalUpgradeRun(command.Command):
                    "execution will be limited to. For example: --limit"
                    " \"compute-0,compute-1,compute-5\".")
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
 
         return parser
 
@@ -143,6 +151,7 @@ class ExternalUpgradeRun(command.Command):
             tags=parsed_args.tags,
             skip_tags=parsed_args.skip_tags,
             limit_hosts=oooutils.playbook_limit_parse(
-                limit_nodes=parsed_args.limit)
+                limit_nodes=parsed_args.limit),
+            forks=parsed_args.ansible_forks
         )
         self.log.info("Completed Overcloud External Upgrade Run.")

tripleoclient/v1/overcloud_update.py

@@ -168,6 +168,14 @@ class UpdateRun(command.Command):
             help=_("Use -y or --yes to skip the confirmation required before "
                    "any update operation. Use this with caution! "),
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
         return parser
 
     def take_action(self, parsed_args):
@@ -210,7 +218,8 @@ class UpdateRun(command.Command):
                 limit_nodes=parsed_args.limit
             ),
             skip_tags=parsed_args.skip_tags,
-            tags=parsed_args.tags
+            tags=parsed_args.tags,
+            forks=parsed_args.ansible_forks
         )
         self.log.info("Completed Overcloud Minor Update Run.")
 

tripleoclient/v1/overcloud_upgrade.py

@@ -204,6 +204,14 @@ class UpgradeRun(command.Command):
                                    "required before any upgrade "
                                    "operation. Use this with caution! ")
                             )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
         return parser
 
     def _validate_skip_tags(self, skip_tags):
@@ -257,7 +265,8 @@ class UpgradeRun(command.Command):
             skip_tags=parsed_args.skip_tags,
             limit_hosts=oooutils.playbook_limit_parse(
                 limit_nodes=parsed_args.limit
-            )
+            ),
+            forks=parsed_args.ansible_forks
         )
         self.log.info("Completed Overcloud Major Upgrade Run.")
 

tripleoclient/v1/tripleo_deploy.py

@@ -1084,6 +1084,14 @@ class Deploy(command.Command):
                    'deployed services are running right after their '
                    'activation. Defaults to False.')
         )
+        parser.add_argument(
+            '--ansible-forks',
+            action='store',
+            default=None,
+            type=int,
+            help=_('The number of Ansible forks to use for the'
+                   ' config-download ansible-playbook command.')
+        )
 
         stack_action_group = parser.add_mutually_exclusive_group()
 
@@ -1326,6 +1334,7 @@ class Deploy(command.Command):
                             workdir=self.ansible_dir,
                             verbosity=utils.playbook_verbosity(self=self),
                             extra_env_variables=extra_env_var,
+                            forks=parsed_args.ansible_forks,
                             **operation)
             is_complete = True
         finally:

tripleoclient/workflows/deployment.py

@@ -278,7 +278,7 @@ def config_download(log, clients, stack, ssh_network=None,
                     ansible_playbook_name='deploy_steps_playbook.yaml',
                     limit_hosts=None, extra_vars=None, inventory_path=None,
                     ssh_user='tripleo-admin', tags=None, skip_tags=None,
-                    deployment_timeout=None):
+                    deployment_timeout=None, forks=None):
     """Run config download.
 
     :param log: Logging object
@@ -491,6 +491,7 @@ def config_download(log, clients, stack, ssh_network=None,
             },
             extra_vars=extra_vars,
             timeout=deployment_timeout,
+            forks=forks
         )
 
     _log_and_print(