[goal] Deprecate the JSON formatted policy file

As per the community goal of migrating the policy file
the format from JSON to YAML[1], we need to do two things:

1. Change the default value of '[oslo_policy] policy_file''
config option from 'policy.json' to 'policy.yaml' with
upgrade checks.

2. Deprecate the JSON formatted policy file on the project side
via warning in doc and releasenotes.

Also replace policy.json to policy.yaml ref from doc and tests.

CONF object needs to be initialized before policy enforcer(). That
need to remove cfg.CONF.unregister_opts from TestAuthUtils cleanup
as this is taken care by cfg.clear() with proper workflow otherwise
it end up with error
"oslo_config.cfg.ArgsAlreadyParsedError: arguments
already parsed: reset before unregistering options"

- https://b132754ee7062a9ab187-9add4719a9922a9385555a8552fc2366.ssl.cf5.rackcdn.com/768520/5/check/openstack-tox-py38/7964354/testr_results.html

[1]https://governance.openstack.org/tc/goals/selected/wallaby/migrate-policy-format-from-json-to-yaml.html

Change-Id: I1b6c6485bc651fd0b87244a68204036dd4aa37f4

doc/source/configuration/sample_policy.rst

@@ -2,6 +2,14 @@
 Murano Sample Policy
 ====================
 
+.. warning::
+
+   JSON formatted policy file is deprecated since Murano 11.0.0 (Wallaby).
+   This `oslopolicy-convert-json-to-yaml`__ tool will migrate your existing
+   JSON-formatted policy file to YAML in a backward-compatible way.
+
+.. __: https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html
+
 The following is a sample murano policy file that has been auto-generated
 from default policy values in code. If you're using the default policies, then
 the maintenance of this file is not necessary, and it should not be copied into

lower-constraints.txt

@@ -68,20 +68,20 @@ os-service-types==1.2.0
 osc-lib==1.14.0
 oslo.cache==1.29.0
 oslo.concurrency==3.26.0
-oslo.config==5.2.0
-oslo.context==2.19.2
+oslo.config==6.8.0
+oslo.context==2.22.0
 oslo.db==4.44.0
 oslo.i18n==3.15.3
 oslo.log==3.36.0
 oslo.messaging==5.29.0
 oslo.middleware==3.31.0
-oslo.policy==1.30.0
+oslo.policy==3.6.0
 oslo.serialization==2.18.0
 oslo.service==1.31.0
-oslo.upgradecheck==0.1.0
-oslo.utils==3.33.0
-oslotest==3.2.0
-packaging==17.1
+oslo.upgradecheck==1.3.0
+oslo.utils==4.5.0
+oslotest==4.4.1
+packaging==20.4
 Paste==2.0.2
 PasteDeploy==1.5.0
 pbr==2.0.0
@@ -121,11 +121,11 @@ python-openstackclient==4.0.0
 python-subunit==1.2.0
 python-swiftclient==3.5.0
 pytz==2018.3
-PyYAML==3.13
+PyYAML==5.1
 repoze.lru==0.7
-requests==2.14.2
+requests==2.20.0
 requestsexceptions==1.4.0
-rfc3986==1.1.0
+rfc3986==1.2.0
 Routes==2.3.1
 semantic-version==2.8.2
 simplejson==3.13.2

murano/cmd/status.py

@@ -15,6 +15,7 @@
 import sys
 
 from oslo_config import cfg
+from oslo_upgradecheck import common_checks
 from oslo_upgradecheck import upgradecheck
 
 from murano.common.i18n import _
@@ -30,17 +31,9 @@ class Checks(upgradecheck.UpgradeCommands):
     and added to _upgrade_checks tuple.
     """
 
-    def _sample_check(self):
-        """This is sample check added to test the upgrade check framework
-
-        It needs to be removed after adding any real upgrade check
-        """
-        return upgradecheck.Result(upgradecheck.Code.SUCCESS, 'Sample detail')
-
     _upgrade_checks = (
-        # Sample check added for now.
-        # Whereas in future real checks must be added here in tuple
-        (_('Sample Check'), _sample_check),
+        (_("Policy File JSON to YAML Migration"),
+         (common_checks.check_policy_json, {'conf': CONF})),
     )
 
 

murano/common/config.py

@@ -17,6 +17,7 @@ from keystoneauth1 import loading as ks_loading
 from oslo_config import cfg
 from oslo_log import log as logging
 from oslo_middleware import cors
+from oslo_policy import opts
 
 from murano.common.i18n import _
 from murano import version
@@ -310,6 +311,23 @@ def parse_args(args=None, usage=None, default_config_files=None):
          default_config_files=default_config_files)
 
 
+def set_lib_defaults():
+    """Update default value for configuration options from other namespace.
+
+    Example, oslo lib config options. This is needed for
+    config generator tool to pick these default value changes.
+    https://docs.openstack.org/oslo.config/latest/cli/
+    generator.html#modifying-defaults-from-other-namespaces
+    """
+
+    set_middleware_defaults()
+
+    # TODO(gmann): Remove setting the default value of config policy_file
+    # once oslo_policy change the default value to 'policy.yaml'.
+    # https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49
+    opts.set_defaults(CONF, 'policy.yaml')
+
+
 def set_middleware_defaults():
     """Update default configuration options for oslo.middleware."""
 

murano/common/policy.py

@@ -16,6 +16,7 @@
 
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_policy import opts
 from oslo_policy import policy
 from webob import exc as exceptions
 
@@ -27,6 +28,12 @@ CONF = cfg.CONF
 
 _ENFORCER = None
 
+# TODO(gmann): Remove setting the default value of config policy_file
+# once oslo_policy change the default value to 'policy.yaml'.
+# https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49
+DEFAULT_POLICY_FILE = 'policy.yaml'
+opts.set_defaults(CONF, DEFAULT_POLICY_FILE)
+
 
 def reset():
     global _ENFORCER

murano/tests/unit/api/base.py

@@ -17,6 +17,7 @@ import logging
 from unittest import mock
 
 import fixtures
+from oslo_config import cfg
 from oslo_utils import timeutils
 import routes
 import urllib
@@ -126,7 +127,7 @@ class ControllerTest(object):
         super(ControllerTest, self).setUp()
 
         self.is_admin = False
-
+        cfg.CONF(args=[], project='murano')
         policy.init(use_conf=False)
         real_policy_check = policy.check
 

murano/tests/unit/cmd/test_status.py

@@ -12,6 +12,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo_config import cfg
 from oslo_upgradecheck.upgradecheck import Code
 
 from murano.cmd import status
@@ -24,7 +25,12 @@ class TestUpgradeChecks(base.MuranoTestCase):
         super(TestUpgradeChecks, self).setUp()
         self.cmd = status.Checks()
 
-    def test__sample_check(self):
-        check_result = self.cmd._sample_check()
-        self.assertEqual(
-            Code.SUCCESS, check_result.code)
+    def test_checks(self):
+        cfg.CONF(args=[], project='murano')
+        for name, func in self.cmd._upgrade_checks:
+            if isinstance(func, tuple):
+                func_name, kwargs = func
+                result = func_name(self, **kwargs)
+            else:
+                result = func(self)
+            self.assertEqual(Code.SUCCESS, result.code)

murano/tests/unit/common/test_auth_utils.py

@@ -32,10 +32,6 @@ class TestAuthUtils(base.MuranoTestCase):
         password_option = ka_loading.get_auth_plugin_conf_options('password')
         cfg.CONF.register_opts(password_option,
                                group=auth_utils.CFG_MURANO_AUTH_GROUP)
-        self.addCleanup(
-            cfg.CONF.unregister_opts,
-            password_option,
-            group=auth_utils.CFG_MURANO_AUTH_GROUP)
         self.addCleanup(mock.patch.stopall)
 
     def _init_mock_cfg(self):

releasenotes/notes/deprecate-json-formatted-policy-file-b41728d03ee008e8.yaml

@@ -0,0 +1,20 @@
+---
+upgrade:
+  - |
+    The default value of ``[oslo_policy] policy_file`` config option has
+    been changed from ``policy.json`` to ``policy.yaml``.
+    Operators who are utilizing customized or previously generated
+    static policy JSON files (which are not needed by default), should
+    generate new policy files or convert them in YAML format. Use the
+    `oslopolicy-convert-json-to-yaml
+    <https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_
+    tool to convert a JSON to YAML formatted policy file in
+    backward compatible way.
+deprecations:
+  - |
+    Use of JSON policy files was deprecated by the ``oslo.policy`` library
+    during the Victoria development cycle. As a result, this deprecation is
+    being noted in the Wallaby cycle with an anticipated future removal of support
+    by ``oslo.policy``. As such operators will need to convert to YAML policy
+    files. Please see the upgrade notes for details on migration of any
+    custom policy files.

requirements.txt

@@ -14,7 +14,7 @@ WebOb>=1.7.1 # MIT
 kombu>=4.6.1 # BSD
 psutil>=3.2.2 # BSD
 netaddr>=0.7.18 # BSD
-PyYAML>=3.13 # MIT
+PyYAML>=5.1 # MIT
 jsonpatch!=1.20,>=1.16 # BSD
 keystoneauth1>=3.8.0 # Apache-2.0
 keystonemiddleware>=4.17.0 # Apache-2.0
@@ -33,17 +33,17 @@ python-neutronclient>=6.7.0 # Apache-2.0
 python-muranoclient>=0.8.2 # Apache-2.0
 python-mistralclient!=3.2.0,>=3.1.0 # Apache-2.0
 oslo.db>=4.44.0 # Apache-2.0
-oslo.config>=5.2.0 # Apache-2.0
+oslo.config>=6.8.0 # Apache-2.0
 oslo.concurrency>=3.26.0 # Apache-2.0
-oslo.context>=2.19.2 # Apache-2.0
-oslo.policy>=1.30.0 # Apache-2.0
+oslo.context>=2.22.0 # Apache-2.0
+oslo.policy>=3.6.0 # Apache-2.0
 oslo.messaging>=5.29.0 # Apache-2.0
 oslo.middleware>=3.31.0 # Apache-2.0
 oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
 oslo.service>=1.31.0 # Apache-2.0
-oslo.utils>=3.33.0 # Apache-2.0
+oslo.utils>=4.5.0 # Apache-2.0
 oslo.i18n>=3.15.3 # Apache-2.0
 oslo.log>=3.36.0 # Apache-2.0
-oslo.upgradecheck>=0.1.0 # Apache-2.0
+oslo.upgradecheck>=1.3.0 # Apache-2.0
 semantic-version>=2.8.2 # BSD
 castellan>=0.18.0 # Apache-2.0

setup.cfg

@@ -67,7 +67,7 @@ oslo.config.opts =
     castellan.config = castellan.options:list_opts
 
 oslo.config.opts.defaults =
-    murano = murano.common.config:set_middleware_defaults
+    murano = murano.common.config:set_lib_defaults
 
 oslo.policy.policies =
     # With the move of default policy in code list_rules returns a list of

test-requirements.txt

@@ -6,14 +6,14 @@ hacking>=3.0.1,<3.1.0 # Apache-2.0
 coverage!=4.4,>=4.0 # Apache-2.0
 fixtures>=3.0.0 # Apache-2.0/BSD
 nose>=1.3.7 # LGPL
-oslotest>=3.2.0 # Apache-2.0
+oslotest>=4.4.1 # Apache-2.0
 sqlalchemy-migrate>=0.11.0 # Apache-2.0
 testrepository>=0.0.18 # Apache-2.0/BSD
 testresources>=2.0.0 # Apache-2.0/BSD
 testscenarios>=0.4 # Apache-2.0/BSD
 pylint==1.4.5 # GPLv2
 pycodestyle>=2.5.0 # MIT License
-requests>=2.14.2 # Apache-2.0
+requests>=2.20.0 # Apache-2.0
 stestr>=2.0.0 # Apache-2.0
 murano-pkg-check>=0.3.0 # Apache-2.0
 bandit>=1.1.0,!=1.6.0 # Apache-2.0