Add common checks for json to yaml policy file migration

As part of community goal[1] for migrating the policy file
from JSON format to YAML we need to add upgrade check to
detect any JSON formatted file. That check needs to be
done in each service side which are deprecating the JSON
format. So to avoid duplicate code, this commit add that
check in oslo_upgradechecks and that can be enabled from
service side during they migrate to YAML format.

Service will enable it like below:

    _upgrade_checks = (
        (_('Policy File JSON to YAML Migration'),
            (common_checks.check_policy_json, {'conf': CONF})),
    )

Also modify the check() method to accept the **kwargs for checks.

[1] https://governance.openstack.org/tc/goals/selected/wallaby/migrate-policy-format-from-json-to-yaml.html

Change-Id: Ifa71c20879c964f794a40b981744d95db4ab1602

lower-constraints.txt

@@ -4,3 +4,5 @@ enum34==1.0.4
 PrettyTable==0.7.1
 oslotest==1.5.1
 stestr==2.0.0
+oslo.serialization==2.21.1
+oslo.utils==4.5.0

oslo_upgradecheck/common_checks.py

@@ -0,0 +1,36 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_utils import fileutils
+
+from oslo_upgradecheck import upgradecheck
+
+"""
+Common checks which can be used by multiple services.
+"""
+
+
+def check_policy_json(self, conf):
+    "Checks to see if policy file is JSON-formatted policy file."
+    msg = ("Your policy file is JSON-formatted which is "
+           "deprecated. You need to switch to YAML-formatted file. "
+           "Use the ``oslopolicy-convert-json-to-yaml`` "
+           "tool to convert the existing JSON-formatted files to "
+           "YAML in a backwards-compatible manner: "
+           "https://docs.openstack.org/oslo.policy/"
+           "latest/cli/oslopolicy-convert-json-to-yaml.html.")
+    status = upgradecheck.Result(upgradecheck.Code.SUCCESS)
+    # Check if policy file exist and is JSON-formatted.
+    policy_path = conf.find_file(conf.oslo_policy.policy_file)
+    if policy_path and fileutils.is_json(policy_path):
+        status = upgradecheck.Result(upgradecheck.Code.FAILURE, msg)
+    return status

oslo_upgradecheck/tests/test_common_checks.py

@@ -0,0 +1,103 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import fixtures
+import os.path
+import tempfile
+import yaml
+
+from oslo_config import cfg
+from oslo_config import fixture as config
+from oslo_serialization import jsonutils
+from oslotest import base
+
+from oslo_upgradecheck import common_checks
+from oslo_upgradecheck import upgradecheck
+
+
+class TestUpgradeCheckPolicyJSON(base.BaseTestCase):
+
+    def setUp(self):
+        super(TestUpgradeCheckPolicyJSON, self).setUp()
+        conf_fixture = self.useFixture(config.Config())
+        conf_fixture.load_raw_values()
+        self.conf = conf_fixture.conf
+        policy_file_opt = cfg.StrOpt('policy_file',
+                                     default='policy.json',
+                                     help='policy file')
+        self.conf.register_opt(policy_file_opt, group="oslo_policy")
+
+        self.cmd = upgradecheck.UpgradeCommands()
+        self.cmd._upgrade_checks = (('Policy File JSON to YAML Migration',
+                                    (common_checks.check_policy_json,
+                                     {'conf': self.conf})),)
+
+        self.data = {
+            'rule_admin': 'True',
+            'rule_admin2': 'is_admin:True'
+        }
+        self.temp_dir = self.useFixture(fixtures.TempDir())
+        fd, self.json_file = tempfile.mkstemp(dir=self.temp_dir.path)
+        fd, self.yaml_file = tempfile.mkstemp(dir=self.temp_dir.path)
+        with open(self.json_file, 'w') as fh:
+            jsonutils.dump(self.data, fh)
+        with open(self.yaml_file, 'w') as fh:
+            yaml.dump(self.data, fh)
+
+        original_search_dirs = cfg._search_dirs
+
+        def fake_search_dirs(dirs, name):
+            dirs.append(self.temp_dir.path)
+            return original_search_dirs(dirs, name)
+
+        mock_search_dir = self.useFixture(
+            fixtures.MockPatch('oslo_config.cfg._search_dirs')).mock
+        mock_search_dir.side_effect = fake_search_dirs
+
+    def test_policy_json_file_fail_upgrade(self):
+        # Test with policy json file full path set in config.
+        self.conf.set_override('policy_file', self.json_file,
+                               group="oslo_policy")
+        self.assertEqual(upgradecheck.Code.FAILURE,
+                         self.cmd.check())
+
+    def test_policy_yaml_file_pass_upgrade(self):
+        # Test with full policy yaml file path set in config.
+        self.conf.set_override('policy_file', self.yaml_file,
+                               group="oslo_policy")
+        self.assertEqual(upgradecheck.Code.SUCCESS,
+                         self.cmd.check())
+
+    def test_no_policy_file_pass_upgrade(self):
+        # Test with no policy file exist, means use policy from code.
+        self.conf.set_override('policy_file', 'non_exist_file',
+                               group="oslo_policy")
+        self.assertEqual(upgradecheck.Code.SUCCESS,
+                         self.cmd.check())
+
+    def test_default_policy_yaml_file_pass_upgrade(self):
+        self.conf.set_override('policy_file', 'policy.yaml',
+                               group="oslo_policy")
+        tmpfilename = os.path.join(self.temp_dir.path, 'policy.yaml')
+        with open(tmpfilename, 'w') as fh:
+            yaml.dump(self.data, fh)
+        self.assertEqual(upgradecheck.Code.SUCCESS,
+                         self.cmd.check())
+
+    def test_old_default_policy_json_file_fail_upgrade(self):
+        self.conf.set_override('policy_file', 'policy.json',
+                               group="oslo_policy")
+        tmpfilename = os.path.join(self.temp_dir.path, 'policy.json')
+        with open(tmpfilename, 'w') as fh:
+            jsonutils.dump(self.data, fh)
+        self.assertEqual(upgradecheck.Code.FAILURE,
+                         self.cmd.check())

oslo_upgradecheck/upgradecheck.py

@@ -99,7 +99,11 @@ class UpgradeCommands(object):
         # This is a list if 2-item tuples for the check name and it's results.
         check_results = []
         for name, func in self._upgrade_checks:
-            result = func(self)
+            if isinstance(func, tuple):
+                func_name, kwargs = func
+                result = func_name(self, **kwargs)
+            else:
+                result = func(self)
             # store the result of the check for the summary table
             check_results.append((name, result))
             # we want to end up with the highest level code of all checks

requirements.txt

@@ -5,3 +5,4 @@
 oslo.config>=5.2.0 # Apache-2.0
 oslo.i18n>=3.15.3 # Apache-2.0
 PrettyTable<0.8,>=0.7.1 # BSD
+oslo.utils>=4.5.0 # Apache-2.0

test-requirements.txt

@@ -6,3 +6,4 @@ hacking>=3.0,<3.1.0 # Apache-2.0
 oslotest>=1.5.1
 stestr>=2.0.0
 pre-commit>=2.6.0 # MIT
+oslo.serialization>=2.21.1 # Apache-2.0