Add kuryr-status utility for upgrade-checkers

This commit adds kuryr-status utility that can be used to check if
upgrade is possible, convert annotations to new format and rollback
those changes if needed.

Implements: blueprint upgrade-checkers

Change-Id: I7a40a68518d7fbba18146b64befb6f585176ec8d

doc/source/installation/index.rst

@@ -33,6 +33,7 @@ This section describes how you can install and configure kuryr-kubernetes
     ports-pool
     services
     ipv6
+    upgrades
     devstack/index
     default_configuration
     trunk_ports

doc/source/installation/upgrades.rst

@@ -0,0 +1,89 @@
+Upgrading kuryr-kubernetes
+===========================
+
+Kuryr-Kubernetes supports standard OpenStack utility for checking upgrade
+is possible and safe:
+
+.. code-block:: bash
+
+    $ kuryr-status upgrade check
+    +---------------------------------------+
+    | Upgrade Check Results                 |
+    +---------------------------------------+
+    | Check: Pod annotations                |
+    | Result: Success                       |
+    | Details: All annotations are updated. |
+    +---------------------------------------+
+
+If any issue will be found, the utility will give you explanation and possible
+remediations. Also note that *Warning* results aren't blocking an upgrade, but
+are worth investigating.
+
+Stein (0.6.x) to T (0.7.x) upgrade
+----------------------------------
+
+In T we want to drop support for old format of Pod annotations (switch was
+motivated by multi-vif support feature implemented in Rocky). To make sure that
+you don't have unsupported Pod annotations you need to run ``kuryr-status
+upgrade check`` utility **before upgrading Kuryr-Kubernetes services to T**.
+
+.. note::
+
+    In case of running Kuryr-Kubernetes containerized you can use ``kubectl
+    exec`` to run kuryr-status
+
+    .. code-block:: bash
+
+        $ kubectl -n kube-system exec -it <controller-pod-name> kuryr-status upgrade check
+
+.. code-block:: bash
+
+    $ kuryr-status upgrade check
+    +---------------------------------------+
+    | Upgrade Check Results                 |
+    +---------------------------------------+
+    | Check: Pod annotations                |
+    | Result: Success                       |
+    | Details: All annotations are updated. |
+    +---------------------------------------+
+
+In case of *Failure* result of *Pod annotations* check you should run
+``kuryr-status upgrade update-annotations`` command and check again:
+
+.. code-block:: bash
+
+    $ kuryr-status upgrade check
+    +----------------------------------------------------------------------+
+    | Upgrade Check Results                                                |
+    +----------------------------------------------------------------------+
+    | Check: Pod annotations                                               |
+    | Result: Failure                                                      |
+    | Details: You have 3 Kuryr pod annotations in old format. You need to |
+    |          run `kuryr-status upgrade update-annotations`               |
+    |          before proceeding with the upgrade.                         |
+    +----------------------------------------------------------------------+
+    $ kuryr-status upgrade update-annotations
+    +-----------------------+--------+
+    | Stat                  | Number |
+    +-----------------------+--------+
+    | Updated annotations   | 3      |
+    +-----------------------+--------+
+    | Malformed annotations | 0      |
+    +-----------------------+--------+
+    | Annotations left      | 0      |
+    +-----------------------+--------+
+    $ kuryr-status upgrade check
+    +---------------------------------------+
+    | Upgrade Check Results                 |
+    +---------------------------------------+
+    | Check: Pod annotations                |
+    | Result: Success                       |
+    | Details: All annotations are updated. |
+    +---------------------------------------+
+
+It's possible that some annotations were somehow malformed. That will generate
+a warning that should be investigated, but isn't blocking upgrading to T
+(it won't make things any worse).
+
+If in any case you need to rollback those changes, there is
+``kuryr-status upgrade downgrade-annotations`` command as well.

kuryr_kubernetes/cmd/status.py

@@ -0,0 +1,266 @@
+# Copyright 2018 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+CLI interface for kuryr status commands.
+"""
+
+from __future__ import print_function
+
+import sys
+import textwrap
+import traceback
+
+import prettytable
+
+import os_vif
+from os_vif.objects import base
+from oslo_config import cfg
+from oslo_serialization import jsonutils
+
+from kuryr_kubernetes import clients
+from kuryr_kubernetes import config
+from kuryr_kubernetes import constants
+from kuryr_kubernetes import exceptions
+from kuryr_kubernetes import objects
+from kuryr_kubernetes.objects import vif
+from kuryr_kubernetes import version
+
+CONF = config.CONF
+
+UPGRADE_CHECK_SUCCESS = 0
+UPGRADE_CHECK_WARNING = 1
+UPGRADE_CHECK_FAILURE = 2
+
+UPGRADE_CHECK_MSG_MAP = {
+    UPGRADE_CHECK_SUCCESS: 'Success',
+    UPGRADE_CHECK_WARNING: 'Warning',
+    UPGRADE_CHECK_FAILURE: 'Failure',
+}
+
+
+class UpgradeCheckResult(object):
+    """Class used for 'kuryr-status upgrade check' results.
+
+    The 'code' attribute is an UpgradeCheckCode enum.
+    The 'details' attribute is a message generally only used for
+    checks that result in a warning or failure code. The details should provide
+    information on what issue was discovered along with any remediation.
+    """
+
+    def __init__(self, code, details=None):
+        super(UpgradeCheckResult, self).__init__()
+        self.code = code
+        self.details = details
+
+    def get_details(self):
+        if self.details is not None:
+            # wrap the text on the details to 60 characters
+            return '\n'.join(textwrap.wrap(self.details, 60,
+                                           subsequent_indent=' ' * 9))
+
+
+class UpgradeCommands(object):
+    def __init__(self):
+        self.check_methods = {
+            'Pod annotations': self._check_annotations,  # Stein
+        }
+        clients.setup_kubernetes_client()
+        self.k8s = clients.get_kubernetes_client()
+
+    def _get_annotation(self, pod):
+        annotations = pod['metadata']['annotations']
+        if constants.K8S_ANNOTATION_VIF not in annotations:
+            # NOTE(dulek): We ignore pods without annotation, those
+            # probably are hostNetworking.
+            return None
+        k_ann = annotations[constants.K8S_ANNOTATION_VIF]
+        k_ann = jsonutils.loads(k_ann)
+        obj = base.VersionedObject.obj_from_primitive(k_ann)
+        return obj
+
+    def _check_annotations(self):
+        old_count = 0
+        malformed_count = 0
+        pods = self.k8s.get('/api/v1/pods')['items']
+        for pod in pods:
+            try:
+                obj = self._get_annotation(pod)
+                if not obj:
+                    # NOTE(dulek): We ignore pods without annotation, those
+                    # probably are hostNetworking.
+                    continue
+            except Exception:
+                # TODO(dulek): We might want to print this exception.
+                malformed_count += 1
+                continue
+
+            if obj.obj_name() != objects.vif.PodState.obj_name():
+                old_count += 1
+
+        if malformed_count == 0 and old_count == 0:
+            return UpgradeCheckResult(0, 'All annotations are updated.')
+        elif malformed_count > 0 and old_count == 0:
+            msg = ('You have %d malformed Kuryr pod annotations in your '
+                   'deployment. This is not blocking the upgrade, but '
+                   'consider investigating it.' % malformed_count)
+            return UpgradeCheckResult(1, msg)
+        elif old_count > 0:
+            msg = ('You have %d Kuryr pod annotations in old format. You need '
+                   'to run `kuryr-status upgrade update-annotations` before '
+                   'proceeding with the upgrade.' % old_count)
+            return UpgradeCheckResult(2, msg)
+
+    def upgrade_check(self):
+        check_results = []
+
+        t = prettytable.PrettyTable(['Upgrade Check Results'],
+                                    hrules=prettytable.ALL)
+        t.align = 'l'
+
+        for name, method in self.check_methods.items():
+            result = method()
+            check_results.append(result)
+            cell = (
+                'Check: %(name)s\n'
+                'Result: %(result)s\n'
+                'Details: %(details)s' %
+                {
+                    'name': name,
+                    'result': UPGRADE_CHECK_MSG_MAP[result.code],
+                    'details': result.get_details(),
+                }
+            )
+            t.add_row([cell])
+        print(t)
+
+        return max(res.code for res in check_results)
+
+    def _convert_annotations(self, test_fn, update_fn):
+        updated_count = 0
+        not_updated_count = 0
+        malformed_count = 0
+        pods = self.k8s.get('/api/v1/pods')['items']
+        for pod in pods:
+            try:
+                obj = self._get_annotation(pod)
+                if not obj:
+                    # NOTE(dulek): We ignore pods without annotation, those
+                    # probably are hostNetworking.
+                    continue
+            except Exception:
+                malformed_count += 1
+                continue
+
+            if test_fn(obj):
+                obj = update_fn(obj)
+                serialized = obj.obj_to_primitive()
+                try:
+                    ann = {
+                        constants.K8S_ANNOTATION_VIF:
+                            jsonutils.dumps(serialized)
+                    }
+                    self.k8s.annotate(
+                        pod['metadata']['selfLink'], ann,
+                        pod['metadata']['resourceVersion'])
+                except exceptions.K8sClientException:
+                    print('Error when updating annotation for pod %s/%s' %
+                          (pod['metadata']['namespace'],
+                           pod['metadata']['name']))
+                    not_updated_count += 1
+
+                updated_count += 1
+
+        t = prettytable.PrettyTable(['Stat', 'Number'],
+                                    hrules=prettytable.ALL)
+        t.align = 'l'
+
+        cells = [['Updated annotations', updated_count],
+                 ['Malformed annotations', malformed_count],
+                 ['Annotations left', not_updated_count]]
+        for cell in cells:
+            t.add_row(cell)
+        print(t)
+
+    def update_annotations(self):
+        def test_fn(obj):
+            return obj.obj_name() != objects.vif.PodState.obj_name()
+
+        def update_fn(obj):
+            return vif.PodState(default_vif=obj)
+
+        self._convert_annotations(test_fn, update_fn)
+
+    def downgrade_annotations(self):
+        def test_fn(obj):
+            return obj.obj_name() == objects.vif.PodState.obj_name()
+
+        def update_fn(obj):
+            return obj.default_vif
+
+        self._convert_annotations(test_fn, update_fn)
+
+
+def print_version():
+    print(version.version_info.version_string())
+
+
+def add_parsers(subparsers):
+    upgrade_cmds = UpgradeCommands()
+
+    upgrade = subparsers.add_parser(
+        'upgrade', help='Actions related to upgrades between releases.')
+    sub = upgrade.add_subparsers()
+
+    check = sub.add_parser('check', help='Check if upgrading is possible.')
+    check.set_defaults(action_fn=upgrade_cmds.upgrade_check)
+
+    ann_update = sub.add_parser(
+        'update-annotations',
+        help='Update annotations in K8s API to newest version.')
+    ann_update.set_defaults(action_fn=upgrade_cmds.update_annotations)
+
+    ann_downgrade = sub.add_parser(
+        'downgrade-annotations',
+        help='Downgrade annotations in K8s API to previous version (useful '
+             'when reverting a failed upgrade).')
+    ann_downgrade.set_defaults(action_fn=upgrade_cmds.downgrade_annotations)
+
+    version_action = subparsers.add_parser('version')
+    version_action.set_defaults(action_fn=print_version)
+
+
+def main():
+    opt = cfg.SubCommandOpt(
+        'category', title='command',
+        description='kuryr-status command or category to execute',
+        handler=add_parsers)
+
+    conf = cfg.ConfigOpts()
+    conf.register_cli_opt(opt)
+    conf(sys.argv[1:])
+
+    os_vif.initialize()
+    objects.register_locally_defined_vifs()
+
+    try:
+        return conf.category.action_fn()
+    except Exception:
+        print('Error:\n%s' % traceback.format_exc())
+        # This is 255 so it's not confused with the upgrade check exit codes.
+        return 255
+
+
+if __name__ == '__main__':
+    main()

kuryr_kubernetes/tests/unit/cmd/test_status.py

@@ -0,0 +1,183 @@
+# Copyright 2018 Red Hat
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from oslo_serialization import jsonutils
+import six
+
+from kuryr_kubernetes.cmd import status
+from kuryr_kubernetes import constants
+from kuryr_kubernetes.objects import vif
+from kuryr_kubernetes.tests import base as test_base
+
+
+class TestStatusCmd(test_base.TestCase):
+    def setUp(self):
+        super(TestStatusCmd, self).setUp()
+        self.cmd = status.UpgradeCommands()
+
+    def test_upgrade_result_get_details(self):
+        res = status.UpgradeCheckResult(0, 'a ' * 50)
+
+        self.assertEqual(
+            (('a ' * 30).rstrip() + '\n' + (' ' * 9) + ('a ' * 20)).rstrip(),
+            res.get_details())
+
+    def test__get_annotation_missing(self):
+        pod = {
+            'metadata': {
+                'annotations': {}
+            }
+        }
+
+        self.assertIsNone(self.cmd._get_annotation(pod))
+
+    def test__get_annotation_existing(self):
+        mock_obj = vif.PodState(
+            default_vif=vif.VIFMacvlanNested(vif_name='foo'))
+
+        pod = {
+            'metadata': {
+                'annotations': {
+                    constants.K8S_ANNOTATION_VIF: jsonutils.dumps(
+                        mock_obj.obj_to_primitive())
+                }
+            }
+        }
+
+        obj = self.cmd._get_annotation(pod)
+        self.assertEqual(mock_obj, obj)
+
+    @mock.patch('sys.stdout', new_callable=six.StringIO)
+    def _test_upgrade_check(self, code, code_name, m_stdout):
+        method_success_m = mock.Mock()
+        method_success_m.return_value = status.UpgradeCheckResult(0, 'foo')
+        method_code_m = mock.Mock()
+        method_code_m.return_value = status.UpgradeCheckResult(code, 'bar')
+
+        self.cmd.check_methods = {'baz': method_success_m,
+                                  'blah': method_code_m}
+        self.assertEqual(code, self.cmd.upgrade_check())
+
+        output = m_stdout.getvalue()
+        self.assertIn('baz', output)
+        self.assertIn('bar', output)
+        self.assertIn('foo', output)
+        self.assertIn('blah', output)
+        self.assertIn('Success', output)
+        self.assertIn(code_name, output)
+
+    def test_upgrade_check_success(self):
+        self._test_upgrade_check(0, 'Success')
+
+    def test_upgrade_check_warning(self):
+        self._test_upgrade_check(1, 'Warning')
+
+    def test_upgrade_check_failure(self):
+        self._test_upgrade_check(2, 'Failure')
+
+    def _test__check_annotations(self, ann_objs, code):
+        pods = {
+            'items': [
+                {
+                    'metadata': {
+                        'annotations': {
+                            constants.K8S_ANNOTATION_VIF: ann
+                        }
+                    }
+                } for ann in ann_objs
+            ]
+        }
+        self.cmd.k8s = mock.Mock(get=mock.Mock(return_value=pods))
+        res = self.cmd._check_annotations()
+        self.assertEqual(code, res.code)
+
+    def test__check_annotations_succeed(self):
+        ann_objs = [
+            vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo')),
+            vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='bar')),
+        ]
+        ann_objs = [jsonutils.dumps(ann.obj_to_primitive())
+                    for ann in ann_objs]
+
+        self._test__check_annotations(ann_objs, 0)
+
+    def test__check_annotations_failure(self):
+        ann_objs = [
+            vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo')),
+            vif.VIFMacvlanNested(vif_name='bar'),
+        ]
+        ann_objs = [jsonutils.dumps(ann.obj_to_primitive())
+                    for ann in ann_objs]
+
+        self._test__check_annotations(ann_objs, 2)
+
+    def test__check_annotations_malformed_and_old(self):
+        ann_objs = [
+            vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo')),
+            vif.VIFMacvlanNested(vif_name='bar'),
+        ]
+        ann_objs = [jsonutils.dumps(ann.obj_to_primitive())
+                    for ann in ann_objs]
+        ann_objs.append('{}')
+
+        self._test__check_annotations(ann_objs, 2)
+
+    def test__check_annotations_malformed(self):
+        ann_objs = [
+            vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo')),
+        ]
+        ann_objs = [jsonutils.dumps(ann.obj_to_primitive())
+                    for ann in ann_objs]
+        ann_objs.append('{}')
+
+        self._test__check_annotations(ann_objs, 1)
+
+    def _test__convert_annotations(self, method, calls):
+        self.cmd.k8s.annotate = mock.Mock()
+
+        ann_objs = [
+            ('foo',
+             vif.PodState(default_vif=vif.VIFMacvlanNested(vif_name='foo'))),
+            ('bar', vif.VIFMacvlanNested(vif_name='bar')),
+        ]
+        ann_objs = [(name, jsonutils.dumps(ann.obj_to_primitive()))
+                    for name, ann in ann_objs]
+
+        pods = {
+            'items': [
+                {
+                    'metadata': {
+                        'annotations': {
+                            constants.K8S_ANNOTATION_VIF: ann
+                        },
+                        'selfLink': name,
+                        'resourceVersion': 1,
+                    }
+                } for name, ann in ann_objs
+            ]
+        }
+        self.cmd.k8s = mock.Mock(get=mock.Mock(return_value=pods))
+        method()
+        for args in calls:
+            self.cmd.k8s.annotate.assert_any_call(*args)
+
+    def test_update_annotations(self):
+        self._test__convert_annotations(self.cmd.update_annotations,
+                                        [('bar', mock.ANY, 1)])
+
+    def test_downgrade_annotations(self):
+        self._test__convert_annotations(self.cmd.downgrade_annotations,
+                                        [('foo', mock.ANY, 1)])

kuryr_kubernetes/version.py

@@ -12,5 +12,4 @@
 
 import pbr.version
 
-version_info = pbr.version.VersionInfo(
+version_info = pbr.version.VersionInfo('kuryr_kubernetes')
-    'kuryr_kubernetes')

releasenotes/notes/stein-upgrade-226c8e7b735701ee.yaml

@@ -0,0 +1,6 @@
+---
+upgrade:
+  - |
+    Before upgrading to T (0.7.x) run ``kuryr-status upgrade check`` to check
+    if upgrade is possible. In case of negative result refer to
+    kuryr-kubernetes documentation for mitigation steps.

requirements.txt

@@ -16,6 +16,7 @@ oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
 oslo.service!=1.28.1,>=1.24.0 # Apache-2.0
 oslo.utils>=3.33.0 # Apache-2.0
 os-vif!=1.8.0,>=1.7.0 # Apache-2.0
+PrettyTable<0.8,>=0.7.2  # BSD
 pyroute2>=0.5.1;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
 retrying!=1.3.0,>=1.2.3 # Apache-2.0
 six>=1.10.0 # MIT

setup.cfg

@@ -31,6 +31,7 @@ console_scripts =
     kuryr-k8s-controller = kuryr_kubernetes.cmd.eventlet.controller:start
     kuryr-daemon = kuryr_kubernetes.cmd.daemon:start
     kuryr-cni = kuryr_kubernetes.cmd.cni:run
+    kuryr-status = kuryr_kubernetes.cmd.status:main
 
 kuryr_kubernetes.vif_translators =
     ovs = kuryr_kubernetes.os_vif_util:neutron_to_osvif_vif_ovs