[goal] Deprecate the JSON formatted policy file

As per the community goal of migrating the policy file
the format from JSON to YAML[1], we need to do two things:

1. Change the default value of '[oslo_policy] policy_file''
config option from 'policy.json' to 'policy.yaml' with
upgrade checks.

2. Deprecate the JSON formatted policy file on the project side
via warning in doc and releasenotes.

Also replace policy.json to policy.yaml ref from doc.

[1]https://governance.openstack.org/tc/goals/selected/wallaby/migrate-policy-format-from-json-to-yaml.html

Change-Id: I81e7ee3243af11ebb3589f530533731b87178a96

designate/cmd/status.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from oslo_upgradecheck import common_checks
 from oslo_upgradecheck import upgradecheck
 from sqlalchemy import MetaData, Table, select, func
 
@@ -43,6 +44,9 @@ class Checks(upgradecheck.UpgradeCommands):
 
     _upgrade_checks = ((_('Duplicate service status'),
                         _duplicate_service_status),
+                       (_('Policy File JSON to YAML Migration'),
+                        (common_checks.check_policy_json,
+                         {'conf': designate.conf.CONF})),
                        )
 
 

designate/common/config.py

@@ -13,6 +13,9 @@
 # under the License.
 
 from oslo_middleware import cors
+from oslo_policy import opts as policy_opts
+
+import designate.conf
 
 
 def set_defaults():
@@ -34,3 +37,8 @@ def set_defaults():
                        'PATCH',
                        'HEAD']
     )
+    # TODO(gmann): Remove setting the default value of config policy_file
+    # once oslo_policy change the default value to 'policy.yaml'.
+    # https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49
+    DEFAULT_POLICY_FILE = 'policy.yaml'
+    policy_opts.set_defaults(designate.conf.CONF, DEFAULT_POLICY_FILE)

designate/policy.py

@@ -25,7 +25,11 @@ from designate.common import policies
 CONF = cfg.CONF
 
 # Add the default policy opts
-opts.set_defaults(CONF)
+# TODO(gmann): Remove setting the default value of config policy_file
+# once oslo_policy change the default value to 'policy.yaml'.
+# https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49
+DEFAULT_POLICY_FILE = 'policy.yaml'
+opts.set_defaults(CONF, DEFAULT_POLICY_FILE)
 
 LOG = logging.getLogger(__name__)
 

designate/tests/unit/mdns/test_service.py

@@ -38,7 +38,8 @@ class MdnsServiceTest(oslotest.base.BaseTestCase):
         self.stdlog = fixtures.StandardLogging()
         self.useFixture(self.stdlog)
 
-        self.useFixture(cfg_fixture.Config(CONF))
+        conf = self.useFixture(cfg_fixture.Config(CONF))
+        conf.conf([], project='designate')
 
         self.service = service.Service()
 

designate/tests/unit/producer/test_service.py

@@ -35,7 +35,8 @@ CONF = cfg.CONF
 @mock.patch.object(service.rpcapi.CentralAPI, 'get_instance', mock.Mock())
 class ProducerTest(oslotest.base.BaseTestCase):
     def setUp(self):
-        self.useFixture(cfg_fixture.Config(CONF))
+        conf = self.useFixture(cfg_fixture.Config(CONF))
+        conf.conf([], project='designate')
 
         service.CONF = RoObject({
             'service:producer': RoObject({

designate/tests/unit/sink/test_notifications.py

@@ -41,6 +41,7 @@ class TestSinkNotification(oslotest.base.BaseTestCase,
             'allowed_event_types', ['compute.instance.create.end'],
             'handler:fake'
         )
+        CONF([], project='designate')
 
         self.context = mock.Mock()
         self.service = service.Service()

designate/tests/unit/test_central/test_basic.py

@@ -229,7 +229,7 @@ class CentralBasic(TestCase):
     def setUp(self):
         super(CentralBasic, self).setUp()
         self.CONF = self.useFixture(cfg_fixture.Config(cfg.CONF)).conf
-
+        self.CONF([], project='designate')
         mock_storage = mock.Mock(spec=designate.storage.base.Storage)
 
         pool_list = objects.PoolList.from_list(
@@ -2196,6 +2196,7 @@ class CentralQuotaTest(unittest.TestCase):
     @patch('designate.central.service.storage')
     @patch('designate.central.service.quota')
     def test_zone_record_quota_allows_lowering_value(self, quota, storage):
+        cfg.CONF([], project='designate')
         service = Service()
         service.storage.count_records.return_value = 10
 

designate/tests/unit/workers/test_service.py

@@ -37,8 +37,8 @@ class TestService(oslotest.base.BaseTestCase):
         super(TestService, self).setUp()
         self.stdlog = fixtures.StandardLogging()
         self.useFixture(self.stdlog)
-        self.useFixture(cfg_fixture.Config(CONF))
-
+        conf = self.useFixture(cfg_fixture.Config(CONF))
+        conf.conf([], project='designate')
         self.context = mock.Mock()
         self.zone = mock.Mock()
         self.service = service.Service()

doc/source/admin/backends/msdns_agent.rst

@@ -89,7 +89,7 @@ Ensure that "policy_file" under the [default] section is set:
 
 .. code-block:: ini
 
-    policy_file = C:\\etc\\designate\\policy.json
+    policy_file = C:\\etc\\designate\\policy.yaml
 
 Start the designate agent using
 (Python 2.7 was installed in the default location C:\\Python27):

doc/source/admin/policy.rst

@@ -2,6 +2,14 @@
 Policy Documentation
 ====================
 
+.. warning::
+
+   JSON formatted policy file is deprecated since Designate 12.0.0 (Wallaby).
+   This `oslopolicy-convert-json-to-yaml`__ tool will migrate your existing
+   JSON-formatted policy file to YAML in a backward-compatible way.
+
+.. __: https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html
+
 The following is an overview of all available policies in Designate. For a
 sample configuration file, refer to :doc:`samples/policy-yaml`.
 

doc/source/admin/quotas.rst

@@ -115,7 +115,7 @@ Per-Tenant via API
 These quotas can be edited via API on a per-tenant basis. An administrator
 can edit quotas for any tenant, but they must supply the
 ``X-Auth-All-Projects`` header, and have permission to use it, they'll also
-need the ``set-quotas`` permission in ``policy.json``. For example, an
+need the ``set-quotas`` permission in ``policy.yaml``. For example, an
 admin setting the zones quota for tenant X would look like:
 
 .. code-block:: http

doc/source/contributor/gmr.rst

@@ -345,7 +345,7 @@ GMR Example
       policy_default_rule = default
       policy_dirs =
         policy.d
-      policy_file = /etc/designate/policy.json
+      policy_file = /etc/designate/policy.yaml
       pool-manager-topic = pool_manager
       publish_errors = False
       pybasedir = /opt/stack/designate

lower-constraints.txt

@@ -70,23 +70,23 @@ os-win==4.1.0
 osc-lib==1.10.0
 oslo.cache==1.29.0
 oslo.concurrency==4.2.0
-oslo.config==5.2.0
+oslo.config==6.8.0
 oslo.context==2.22.0
 oslo.db==8.3.0
 oslo.i18n==3.20.0
 oslo.log==4.3.0
 oslo.messaging==12.4.0
 oslo.middleware==3.31.0
-oslo.policy==2.1.0
+oslo.policy==3.6.0
 oslo.reports==1.18.0
 oslo.rootwrap==5.8.0
 oslo.serialization==2.25.0
 oslo.service==1.31.0
-oslo.upgradecheck==0.1.0
-oslo.utils==3.37.0
+oslo.upgradecheck==1.3.0
+oslo.utils==4.5.0
 oslo.versionedobjects==1.31.2
 oslotest==3.2.0
-packaging==17.1
+packaging==20.4
 paramiko==2.7.1
 Paste==2.0.2
 PasteDeploy==1.5.0
@@ -115,13 +115,13 @@ python-mimeparse==1.6.0
 python-neutronclient==6.7.0
 python-subunit==1.2.0
 pytz==2018.3
-PyYAML==3.13
+PyYAML==5.1
 repoze.lru==0.7
 requests-mock==1.2.0
-requests==2.14.2
+requests==2.23.0
 requestsexceptions==1.4.0
 restructuredtext-lint==1.1.3
-rfc3986==1.1.0
+rfc3986==1.2.0
 Routes==2.4.1
 simplejson==3.13.2
 six==1.11.0

releasenotes/notes/deprecate-json-formatted-policy-file-a6a464a8d35e02a5.yaml

@@ -0,0 +1,20 @@
+---
+upgrade:
+  - |
+    The default value of ``[oslo_policy] policy_file`` config option has
+    been changed from ``policy.json`` to ``policy.yaml``.
+    Operators who are utilizing customized or previously generated
+    static policy JSON files (which are not needed by default), should
+    generate new policy files or convert them in YAML format. Use the
+    `oslopolicy-convert-json-to-yaml
+    <https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html>`_
+    tool to convert a JSON to YAML formatted policy file in
+    backward compatible way.
+deprecations:
+  - |
+    Use of JSON policy files was deprecated by the ``oslo.policy`` library
+    during the Victoria development cycle. As a result, this deprecation is
+    being noted in the Wallaby cycle with an anticipated future removal of support
+    by ``oslo.policy``. As such operators will need to convert to YAML policy
+    files. Please see the upgrade notes for details on migration of any
+    custom policy files.

requirements.txt

@@ -10,7 +10,7 @@ jsonschema>=3.2.0 # MIT
 keystoneauth1>=3.4.0 # Apache-2.0
 keystonemiddleware>=4.17.0 # Apache-2.0
 netaddr>=0.7.18 # BSD
-oslo.config>=5.2.0 # Apache-2.0
+oslo.config>=6.8.0 # Apache-2.0
 oslo.concurrency>=4.2.0 # Apache-2.0
 oslo.messaging>=12.4.0 # Apache-2.0
 oslo.middleware>=3.31.0 # Apache-2.0
@@ -19,8 +19,8 @@ oslo.reports>=1.18.0 # Apache-2.0
 oslo.rootwrap>=5.8.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0
 oslo.service>=1.31.0 # Apache-2.0
-oslo.upgradecheck>=0.1.0
-oslo.utils>=3.37.0 # Apache-2.0
+oslo.upgradecheck>=1.3.0
+oslo.utils>=4.5.0 # Apache-2.0
 oslo.versionedobjects>=1.31.2 # Apache-2.0
 Paste>=2.0.2 # MIT
 PasteDeploy>=1.5.0 # MIT
@@ -28,7 +28,7 @@ pbr>=3.1.1 # Apache-2.0
 pecan!=1.0.2,!=1.0.3,!=1.0.4,!=1.2,>=1.0.0 # BSD
 python-designateclient>=2.12.0 # Apache-2.0
 python-neutronclient>=6.7.0 # Apache-2.0
-requests>=2.14.2 # Apache-2.0
+requests>=2.23.0 # Apache-2.0
 tenacity>=6.0.0 # Apache-2.0
 six>=1.11.0 # MIT
 SQLAlchemy>=1.2.19 # MIT
@@ -40,7 +40,7 @@ dnspython>=1.16.0  # http://www.dnspython.org/LICENSE
 oslo.db>=8.3.0 # Apache-2.0
 oslo.i18n>=3.20.0 # Apache-2.0
 oslo.context>=2.22.0 # Apache-2.0
-oslo.policy>=2.1.0 # Apache-2.0
+oslo.policy>=3.6.0 # Apache-2.0
 Werkzeug>=0.9 # BSD License
 python-memcached>=1.56 # PSF
 tooz>=1.58.0 # Apache-2.0