Use undercloud upgrade to update all required pkgs

Rather than require the end user to manually update packages prior to
kicking off the undercloud process we can do it ourselves by
implementing a flag to skip the update and invoking the upgrade after
the packages have been updated.

Change-Id: Idda6387922adeb182afd11cb0d692d1fcceff9a8
Related-Bug: #1869776
(cherry picked from commit 7d1b738910)

tripleoclient/constants.py

@@ -170,6 +170,7 @@ EXPORT_PASSWORD_EXCLUDE_PATTERNS = [
 # Package that need to be to the latest before undercloud
 # update/update/ffwd.
 UNDERCLOUD_EXTRA_PACKAGES = [
+    "openstack-tripleo-common",
     "openstack-tripleo-heat-templates",
     "openstack-tripleo-validations",
     "tripleo-ansible"

tripleoclient/tests/v1/undercloud/test_install_upgrade.py

@@ -17,6 +17,7 @@ import fixtures
 import json
 import mock
 import os
+import sys
 
 from jinja2 import Template
 
@@ -563,6 +564,7 @@ class TestUndercloudUpgrade(TestPluginV1):
         app_args.verbose_level = 1
         self.cmd = undercloud.UpgradeUndercloud(self.app, app_args)
 
+    @mock.patch.object(sys, 'executable', 'python2')
     # TODO(cjeanner) drop once we have proper oslo.privsep
     @mock.patch('getpass.getuser', return_value='stack')
     @mock.patch('shutil.copy')
@@ -582,12 +584,64 @@ class TestUndercloudUpgrade(TestPluginV1):
         self.cmd.take_action(parsed_args)
         mock_run_command.assert_called_with(
             ['sudo', 'yum', 'upgrade', '-y',
+             'python2-tripleoclient',
+             'openstack-tripleo-common',
              'openstack-tripleo-heat-templates',
              'openstack-tripleo-validations',
              'tripleo-ansible'],
             name='Update extra packages'
         )
+        mock_subprocess.assert_called_with([
+            'openstack', 'undercloud', 'upgrade', '--skip-package-updates',
+            '--no-validations'])
 
+    @mock.patch.object(sys, 'executable', 'python3')
+    # TODO(cjeanner) drop once we have proper oslo.privsep
+    @mock.patch('os.geteuid', return_value=1001)
+    @mock.patch('getpass.getuser', return_value='stack')
+    @mock.patch('shutil.copy')
+    @mock.patch('os.mkdir')
+    @mock.patch('tripleoclient.utils.write_env_file', autospec=True)
+    @mock.patch('subprocess.check_call', autospec=True)
+    @mock.patch('tripleoclient.utils.run_command', autospec=True)
+    def test_undercloud_upgrade_all_opts(self, mock_run_command,
+                                         mock_subprocess,
+                                         mock_wr,
+                                         mock_os, mock_copy, mock_user,
+                                         mock_getuid):
+        arglist = ['--force-stack-update', '--no-validations',
+                   '--inflight-validations', '--dry-run', '--yes']
+        verifylist = []
+        self.cmd.app_args.verbose_level = 2
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        # DisplayCommandBase.take_action() returns two tuples
+        self.cmd.take_action(parsed_args)
+        mock_run_command.assert_not_called()
+        mock_subprocess.assert_called_with([
+            'openstack', 'undercloud', 'upgrade', '--skip-package-updates',
+            '--force-stack-update', '--no-validations',
+            '--inflight-validations', '--dry-run', '--yes', '--debug'])
+
+    # TODO(cjeanner) drop once we have proper oslo.privsep
+    @mock.patch('os.geteuid', return_value=1001)
+    @mock.patch('getpass.getuser', return_value='stack')
+    @mock.patch('shutil.copy')
+    @mock.patch('os.mkdir')
+    @mock.patch('tripleoclient.utils.write_env_file', autospec=True)
+    @mock.patch('subprocess.check_call', autospec=True)
+    @mock.patch('tripleoclient.utils.run_command', autospec=True)
+    def test_undercloud_upgrade_no_pkgs(self, mock_run_command,
+                                        mock_subprocess,
+                                        mock_wr,
+                                        mock_os, mock_copy, mock_user,
+                                        mock_getuid):
+        arglist = ['--no-validations', '--skip-package-updates']
+        verifylist = []
+        parsed_args = self.check_parser(self.cmd, arglist, verifylist)
+
+        # DisplayCommandBase.take_action() returns two tuples
+        self.cmd.take_action(parsed_args)
         mock_subprocess.assert_called_with(
             ['sudo', '--preserve-env', 'openstack', 'tripleo', 'deploy',
              '--standalone', '--standalone-role', 'Undercloud', '--stack',
@@ -648,21 +702,13 @@ class TestUndercloudUpgrade(TestPluginV1):
                                                   mock_subprocess,
                                                   mock_wr, mock_os,
                                                   mock_copy, mock_user):
-        arglist = ['--no-validations']
+        arglist = ['--no-validations', '--skip-package-updates']
         verifylist = []
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
         # DisplayCommandBase.take_action() returns two tuples
         self.cmd.take_action(parsed_args)
 
-        mock_run_command.assert_called_with(
-            ['sudo', 'yum', 'upgrade', '-y',
-             'openstack-tripleo-heat-templates',
-             'openstack-tripleo-validations',
-             'tripleo-ansible'],
-            name='Update extra packages'
-        )
-
         mock_subprocess.assert_called_with(
             ['sudo', '--preserve-env', 'openstack', 'tripleo', 'deploy',
              '--standalone', '--standalone-role', 'Undercloud', '--stack',
@@ -722,21 +768,13 @@ class TestUndercloudUpgrade(TestPluginV1):
                                                mock_subprocess,
                                                mock_wr, mock_os,
                                                mock_copy, mock_user):
-        arglist = ['--no-validations']
+        arglist = ['--no-validations', '--skip-package-updates']
         verifylist = []
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
         # DisplayCommandBase.take_action() returns two tuples
         self.cmd.take_action(parsed_args)
 
-        mock_run_command.assert_called_with(
-            ['sudo', 'yum', 'upgrade', '-y',
-             'openstack-tripleo-heat-templates',
-             'openstack-tripleo-validations',
-             'tripleo-ansible'],
-            name='Update extra packages'
-        )
-
         mock_subprocess.assert_called_with(
             ['sudo', '--preserve-env', 'openstack', 'tripleo', 'deploy',
              '--standalone', '--standalone-role', 'Undercloud', '--stack',
@@ -795,22 +833,14 @@ class TestUndercloudUpgrade(TestPluginV1):
     def test_undercloud_upgrade_with_heat_and_yes(self, mock_run_command,
                                                   mock_subprocess,
                                                   mock_wr, mock_os,
-                                                  mock_user, mock_copy):
+                                                  mock_copy, mock_user):
-        arglist = ['--no-validations', '-y']
+        arglist = ['--no-validations', '-y', '--skip-package-updates']
         verifylist = []
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
         # DisplayCommandBase.take_action() returns two tuples
         self.cmd.take_action(parsed_args)
 
-        mock_run_command.assert_called_with(
-            ['sudo', 'yum', 'upgrade', '-y',
-             'openstack-tripleo-heat-templates',
-             'openstack-tripleo-validations',
-             'tripleo-ansible'],
-            name='Update extra packages'
-        )
-
         mock_subprocess.assert_called_with(
             ['sudo', '--preserve-env', 'openstack', 'tripleo', 'deploy',
              '--standalone', '--standalone-role', 'Undercloud', '--stack',
@@ -871,7 +901,7 @@ class TestUndercloudUpgrade(TestPluginV1):
                                                     mock_subprocess,
                                                     mock_wr, mock_os,
                                                     mock_copy, mock_user):
-        arglist = ['--no-validations']
+        arglist = ['--no-validations', '--skip-package-updates']
         verifylist = []
         parsed_args = self.check_parser(self.cmd, arglist, verifylist)
 
@@ -881,14 +911,6 @@ class TestUndercloudUpgrade(TestPluginV1):
         self.cmd.take_action(parsed_args)
         self.cmd.app_args.verbose_level = old_verbose
 
-        mock_run_command.assert_called_with(
-            ['sudo', 'yum', 'upgrade', '-y',
-             'openstack-tripleo-heat-templates',
-             'openstack-tripleo-validations',
-             'tripleo-ansible'],
-            name='Update extra packages'
-        )
-
         mock_subprocess.assert_called_with(
             ['sudo', '--preserve-env', 'openstack', 'tripleo', 'deploy',
              '--standalone', '--standalone-role', 'Undercloud', '--stack',

tripleoclient/v1/undercloud.py

@@ -19,6 +19,7 @@ import argparse
 import logging
 import os
 import subprocess
+import sys
 
 from openstackclient.i18n import _
 
@@ -165,41 +166,55 @@ class UpgradeUndercloud(InstallUndercloud):
     log = logging.getLogger(__name__ + ".UpgradeUndercloud")
     osloconfig = cfg.CONF
 
+    def get_parser(self, prog_name):
+        parser = super(UpgradeUndercloud, self).get_parser(prog_name)
+        parser.add_argument('--skip-package-updates',
+                            dest='skip_package_updates',
+                            action='store_true',
+                            default=False,
+                            help=_("Flag to skip the package update when "
+                                   "performing upgrades and updates"),
+                            )
+        return parser
+
     def _update_extra_packages(self, packages=[], dry_run=False):
         """Necessary packages to be updated before undercloud upgrade."""
 
-        cmd = []
+        if not packages:
-        if packages:
+            return
+
         cmd = ['sudo', 'yum', 'upgrade', '-y'] + packages
 
-        if cmd:
         if not dry_run:
-                self.log.warning(
+            self.log.warning("Updating necessary packages: {}".format(
-                    "Updating necessary packages: {}\n{}".format(
+                             " ".join(packages)))
-                        " ".join(packages),
-                        ("Note that tripleoclient and tripleo-common "
-                         "still need to be updated manually.")
-                    ),
-                )
             output = utils.run_command(cmd, name="Update extra packages")
             self.log.warning("{}".format(output))
         else:
-                self.log.warning(
+            self.log.warning("Would update necessary packages: {}".format(
-                    "Would update necessary packages: {}".format(" ".join(cmd))
+                " ".join(cmd)))
-                )
 
-    def take_action(self, parsed_args):
+    def _invoke_self(self, parsed_args):
-        # Fetch configuration used to add logging to a file
+        cmd = ['openstack', 'undercloud', 'upgrade', '--skip-package-updates']
-        utils.load_config(self.osloconfig, constants.UNDERCLOUD_CONF_PATH)
+        opts = {'force_stack_update': '--force-stack-update',
-        utils.configure_logging(self.log, self.app_args.verbose_level,
+                'no_validations': '--no-validations',
-                                self.osloconfig['undercloud_log_file'])
+                'inflight': '--inflight-validations',
-        self.log.debug("take action(%s)" % parsed_args)
+                'dry_run': '--dry-run',
-
+                'yes': '--yes'}
-        utils.ensure_run_as_normal_user()
+        args = vars(parsed_args)
-
+        for k, v in opts.items():
-        self._update_extra_packages(constants.UNDERCLOUD_EXTRA_PACKAGES,
+            if args[k]:
-                                    parsed_args.dry_run)
+                cmd.append(v)
+        # handle --debug
+        if self.app_args.verbose_level > 1:
+            cmd.append('--debug')
+        try:
+            subprocess.check_call(cmd)
+        except Exception as e:
+            self.log.error(e)
+            raise exceptions.DeploymentError(e)
 
+    def _run_upgrade(self, parsed_args):
         cmd = undercloud_config.\
             prepare_undercloud_deploy(
                 upgrade=True,
@@ -224,3 +239,27 @@ class UpgradeUndercloud(InstallUndercloud):
                 self.log.error(UNDERCLOUD_FAILURE_MESSAGE)
                 self.log.error(e)
                 raise exceptions.DeploymentError(e)
+
+    def take_action(self, parsed_args):
+        # Fetch configuration used to add logging to a file
+        utils.load_config(self.osloconfig, constants.UNDERCLOUD_CONF_PATH)
+        utils.configure_logging(self.log, self.app_args.verbose_level,
+                                self.osloconfig['undercloud_log_file'])
+        self.log.debug("take action(%s)" % parsed_args)
+
+        utils.ensure_run_as_normal_user()
+
+        if not parsed_args.skip_package_updates:
+            print("executable is {}".format(sys.executable))
+            if ('python3' in sys.executable):
+                pyver = '3'
+            else:
+                pyver = '2'
+            client_pkgs = [
+                "python{}-tripleoclient".format(pyver),
+            ]
+            pkgs = client_pkgs + constants.UNDERCLOUD_EXTRA_PACKAGES
+            self._update_extra_packages(pkgs, parsed_args.dry_run)
+            self._invoke_self(parsed_args)
+        else:
+            self._run_upgrade(parsed_args)