From 1c87ebf9a8a79d990e4a1ba44452b50efae77883 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Fri, 25 Dec 2020 15:46:17 -0600 Subject: [PATCH] [goal] Deprecate the JSON formatted policy file As per the community goal of migrating the policy file the format from JSON to YAML[1], we need to do two things: 1. Change the default value of '[oslo_policy] policy_file'' config option from 'policy.json' to 'policy.yaml' with upgrade checks. 2. Deprecate the JSON formatted policy file on the project side via warning in doc and releasenotes. Also replace policy.json to policy.yaml ref from doc and tests. CONF object needs to be initialized before policy enforcer(). That need to remove cfg.CONF.unregister_opts from TestAuthUtils cleanup as this is taken care by cfg.clear() with proper workflow otherwise it end up with error "oslo_config.cfg.ArgsAlreadyParsedError: arguments already parsed: reset before unregistering options" - https://b132754ee7062a9ab187-9add4719a9922a9385555a8552fc2366.ssl.cf5.rackcdn.com/768520/5/check/openstack-tox-py38/7964354/testr_results.html [1]https://governance.openstack.org/tc/goals/selected/wallaby/migrate-policy-format-from-json-to-yaml.html Change-Id: I1b6c6485bc651fd0b87244a68204036dd4aa37f4 --- doc/source/configuration/sample_policy.rst | 8 ++++++++ lower-constraints.txt | 20 +++++++++---------- murano/cmd/status.py | 13 +++--------- murano/common/config.py | 18 +++++++++++++++++ murano/common/policy.py | 7 +++++++ murano/tests/unit/api/base.py | 3 ++- murano/tests/unit/cmd/test_status.py | 14 +++++++++---- murano/tests/unit/common/test_auth_utils.py | 4 ---- ...ormatted-policy-file-b41728d03ee008e8.yaml | 20 +++++++++++++++++++ requirements.txt | 12 +++++------ setup.cfg | 2 +- test-requirements.txt | 4 ++-- 12 files changed, 87 insertions(+), 38 deletions(-) create mode 100644 releasenotes/notes/deprecate-json-formatted-policy-file-b41728d03ee008e8.yaml diff --git a/doc/source/configuration/sample_policy.rst b/doc/source/configuration/sample_policy.rst index b34a463ba..67df03a15 100644 --- a/doc/source/configuration/sample_policy.rst +++ b/doc/source/configuration/sample_policy.rst @@ -2,6 +2,14 @@ Murano Sample Policy ==================== +.. warning:: + + JSON formatted policy file is deprecated since Murano 11.0.0 (Wallaby). + This `oslopolicy-convert-json-to-yaml`__ tool will migrate your existing + JSON-formatted policy file to YAML in a backward-compatible way. + +.. __: https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html + The following is a sample murano policy file that has been auto-generated from default policy values in code. If you're using the default policies, then the maintenance of this file is not necessary, and it should not be copied into diff --git a/lower-constraints.txt b/lower-constraints.txt index de4efc8bd..5ae4f6603 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -68,20 +68,20 @@ os-service-types==1.2.0 osc-lib==1.14.0 oslo.cache==1.29.0 oslo.concurrency==3.26.0 -oslo.config==5.2.0 -oslo.context==2.19.2 +oslo.config==6.8.0 +oslo.context==2.22.0 oslo.db==4.44.0 oslo.i18n==3.15.3 oslo.log==3.36.0 oslo.messaging==5.29.0 oslo.middleware==3.31.0 -oslo.policy==1.30.0 +oslo.policy==3.6.0 oslo.serialization==2.18.0 oslo.service==1.31.0 -oslo.upgradecheck==0.1.0 -oslo.utils==3.33.0 -oslotest==3.2.0 -packaging==17.1 +oslo.upgradecheck==1.3.0 +oslo.utils==4.5.0 +oslotest==4.4.1 +packaging==20.4 Paste==2.0.2 PasteDeploy==1.5.0 pbr==2.0.0 @@ -121,11 +121,11 @@ python-openstackclient==4.0.0 python-subunit==1.2.0 python-swiftclient==3.5.0 pytz==2018.3 -PyYAML==3.13 +PyYAML==5.1 repoze.lru==0.7 -requests==2.14.2 +requests==2.20.0 requestsexceptions==1.4.0 -rfc3986==1.1.0 +rfc3986==1.2.0 Routes==2.3.1 semantic-version==2.8.2 simplejson==3.13.2 diff --git a/murano/cmd/status.py b/murano/cmd/status.py index d4eed88eb..5c48b3896 100644 --- a/murano/cmd/status.py +++ b/murano/cmd/status.py @@ -15,6 +15,7 @@ import sys from oslo_config import cfg +from oslo_upgradecheck import common_checks from oslo_upgradecheck import upgradecheck from murano.common.i18n import _ @@ -30,17 +31,9 @@ class Checks(upgradecheck.UpgradeCommands): and added to _upgrade_checks tuple. """ - def _sample_check(self): - """This is sample check added to test the upgrade check framework - - It needs to be removed after adding any real upgrade check - """ - return upgradecheck.Result(upgradecheck.Code.SUCCESS, 'Sample detail') - _upgrade_checks = ( - # Sample check added for now. - # Whereas in future real checks must be added here in tuple - (_('Sample Check'), _sample_check), + (_("Policy File JSON to YAML Migration"), + (common_checks.check_policy_json, {'conf': CONF})), ) diff --git a/murano/common/config.py b/murano/common/config.py index 0b5029408..14dfa1326 100644 --- a/murano/common/config.py +++ b/murano/common/config.py @@ -17,6 +17,7 @@ from keystoneauth1 import loading as ks_loading from oslo_config import cfg from oslo_log import log as logging from oslo_middleware import cors +from oslo_policy import opts from murano.common.i18n import _ from murano import version @@ -310,6 +311,23 @@ def parse_args(args=None, usage=None, default_config_files=None): default_config_files=default_config_files) +def set_lib_defaults(): + """Update default value for configuration options from other namespace. + + Example, oslo lib config options. This is needed for + config generator tool to pick these default value changes. + https://docs.openstack.org/oslo.config/latest/cli/ + generator.html#modifying-defaults-from-other-namespaces + """ + + set_middleware_defaults() + + # TODO(gmann): Remove setting the default value of config policy_file + # once oslo_policy change the default value to 'policy.yaml'. + # https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49 + opts.set_defaults(CONF, 'policy.yaml') + + def set_middleware_defaults(): """Update default configuration options for oslo.middleware.""" diff --git a/murano/common/policy.py b/murano/common/policy.py index a648aabfb..f20a890e3 100644 --- a/murano/common/policy.py +++ b/murano/common/policy.py @@ -16,6 +16,7 @@ from oslo_config import cfg from oslo_log import log as logging +from oslo_policy import opts from oslo_policy import policy from webob import exc as exceptions @@ -27,6 +28,12 @@ CONF = cfg.CONF _ENFORCER = None +# TODO(gmann): Remove setting the default value of config policy_file +# once oslo_policy change the default value to 'policy.yaml'. +# https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49 +DEFAULT_POLICY_FILE = 'policy.yaml' +opts.set_defaults(CONF, DEFAULT_POLICY_FILE) + def reset(): global _ENFORCER diff --git a/murano/tests/unit/api/base.py b/murano/tests/unit/api/base.py index 99d675f12..c6c9f32e5 100644 --- a/murano/tests/unit/api/base.py +++ b/murano/tests/unit/api/base.py @@ -17,6 +17,7 @@ import logging from unittest import mock import fixtures +from oslo_config import cfg from oslo_utils import timeutils import routes import urllib @@ -126,7 +127,7 @@ class ControllerTest(object): super(ControllerTest, self).setUp() self.is_admin = False - + cfg.CONF(args=[], project='murano') policy.init(use_conf=False) real_policy_check = policy.check diff --git a/murano/tests/unit/cmd/test_status.py b/murano/tests/unit/cmd/test_status.py index f9cef9a1a..a39928b6c 100644 --- a/murano/tests/unit/cmd/test_status.py +++ b/murano/tests/unit/cmd/test_status.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo_config import cfg from oslo_upgradecheck.upgradecheck import Code from murano.cmd import status @@ -24,7 +25,12 @@ class TestUpgradeChecks(base.MuranoTestCase): super(TestUpgradeChecks, self).setUp() self.cmd = status.Checks() - def test__sample_check(self): - check_result = self.cmd._sample_check() - self.assertEqual( - Code.SUCCESS, check_result.code) + def test_checks(self): + cfg.CONF(args=[], project='murano') + for name, func in self.cmd._upgrade_checks: + if isinstance(func, tuple): + func_name, kwargs = func + result = func_name(self, **kwargs) + else: + result = func(self) + self.assertEqual(Code.SUCCESS, result.code) diff --git a/murano/tests/unit/common/test_auth_utils.py b/murano/tests/unit/common/test_auth_utils.py index f64189d69..254f4e1f2 100644 --- a/murano/tests/unit/common/test_auth_utils.py +++ b/murano/tests/unit/common/test_auth_utils.py @@ -32,10 +32,6 @@ class TestAuthUtils(base.MuranoTestCase): password_option = ka_loading.get_auth_plugin_conf_options('password') cfg.CONF.register_opts(password_option, group=auth_utils.CFG_MURANO_AUTH_GROUP) - self.addCleanup( - cfg.CONF.unregister_opts, - password_option, - group=auth_utils.CFG_MURANO_AUTH_GROUP) self.addCleanup(mock.patch.stopall) def _init_mock_cfg(self): diff --git a/releasenotes/notes/deprecate-json-formatted-policy-file-b41728d03ee008e8.yaml b/releasenotes/notes/deprecate-json-formatted-policy-file-b41728d03ee008e8.yaml new file mode 100644 index 000000000..c9c530004 --- /dev/null +++ b/releasenotes/notes/deprecate-json-formatted-policy-file-b41728d03ee008e8.yaml @@ -0,0 +1,20 @@ +--- +upgrade: + - | + The default value of ``[oslo_policy] policy_file`` config option has + been changed from ``policy.json`` to ``policy.yaml``. + Operators who are utilizing customized or previously generated + static policy JSON files (which are not needed by default), should + generate new policy files or convert them in YAML format. Use the + `oslopolicy-convert-json-to-yaml + `_ + tool to convert a JSON to YAML formatted policy file in + backward compatible way. +deprecations: + - | + Use of JSON policy files was deprecated by the ``oslo.policy`` library + during the Victoria development cycle. As a result, this deprecation is + being noted in the Wallaby cycle with an anticipated future removal of support + by ``oslo.policy``. As such operators will need to convert to YAML policy + files. Please see the upgrade notes for details on migration of any + custom policy files. diff --git a/requirements.txt b/requirements.txt index d229ddabc..cf96071f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ WebOb>=1.7.1 # MIT kombu>=4.6.1 # BSD psutil>=3.2.2 # BSD netaddr>=0.7.18 # BSD -PyYAML>=3.13 # MIT +PyYAML>=5.1 # MIT jsonpatch!=1.20,>=1.16 # BSD keystoneauth1>=3.8.0 # Apache-2.0 keystonemiddleware>=4.17.0 # Apache-2.0 @@ -33,17 +33,17 @@ python-neutronclient>=6.7.0 # Apache-2.0 python-muranoclient>=0.8.2 # Apache-2.0 python-mistralclient!=3.2.0,>=3.1.0 # Apache-2.0 oslo.db>=4.44.0 # Apache-2.0 -oslo.config>=5.2.0 # Apache-2.0 +oslo.config>=6.8.0 # Apache-2.0 oslo.concurrency>=3.26.0 # Apache-2.0 -oslo.context>=2.19.2 # Apache-2.0 -oslo.policy>=1.30.0 # Apache-2.0 +oslo.context>=2.22.0 # Apache-2.0 +oslo.policy>=3.6.0 # Apache-2.0 oslo.messaging>=5.29.0 # Apache-2.0 oslo.middleware>=3.31.0 # Apache-2.0 oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 oslo.service>=1.31.0 # Apache-2.0 -oslo.utils>=3.33.0 # Apache-2.0 +oslo.utils>=4.5.0 # Apache-2.0 oslo.i18n>=3.15.3 # Apache-2.0 oslo.log>=3.36.0 # Apache-2.0 -oslo.upgradecheck>=0.1.0 # Apache-2.0 +oslo.upgradecheck>=1.3.0 # Apache-2.0 semantic-version>=2.8.2 # BSD castellan>=0.18.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 38e84c3f1..375e6f8be 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ oslo.config.opts = castellan.config = castellan.options:list_opts oslo.config.opts.defaults = - murano = murano.common.config:set_middleware_defaults + murano = murano.common.config:set_lib_defaults oslo.policy.policies = # With the move of default policy in code list_rules returns a list of diff --git a/test-requirements.txt b/test-requirements.txt index 871265206..3ae121734 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,14 +6,14 @@ hacking>=3.0.1,<3.1.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD nose>=1.3.7 # LGPL -oslotest>=3.2.0 # Apache-2.0 +oslotest>=4.4.1 # Apache-2.0 sqlalchemy-migrate>=0.11.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD testresources>=2.0.0 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD pylint==1.4.5 # GPLv2 pycodestyle>=2.5.0 # MIT License -requests>=2.14.2 # Apache-2.0 +requests>=2.20.0 # Apache-2.0 stestr>=2.0.0 # Apache-2.0 murano-pkg-check>=0.3.0 # Apache-2.0 bandit>=1.1.0,!=1.6.0 # Apache-2.0