JSON serialize mappingproxy types

This test update is modelled after the devstack base job which sets
devstack_local_conf variables with many nested layers of dicts which
child jobs never override.

In Job._deepUpdate, only coerce to dict if needed to perform the update.

Create a new json serializer helper which can handle serializing
mappingproxy types, and let that handle all of the coercion upon
serialization.

Change-Id: Ifa3d1860e10e98f3a327be689b996ea194f5e7ab
Co-Authored-By: James E. Blair <jeblair@redhat.com>

tests/fixtures/layouts/job-vars.yaml

@@ -25,6 +25,9 @@
       deep:
         override: 0
         parent: 0
+        more:
+          nesting:
+            for: raisins
       override: 0
       child1override: 0
       parent: 0

zuul/executor/client.py

@@ -22,6 +22,7 @@ from uuid import uuid4
 
 import zuul.model
 from zuul.lib.config import get_default
+from zuul.lib.jsonutil import json_dumps
 from zuul.model import Build
 
 
@@ -297,7 +298,7 @@ class ExecutorClient(object):
             self.sched.onBuildCompleted(build, 'SUCCESS', {})
             return build
 
-        gearman_job = gear.TextJob('executor:execute', json.dumps(params),
+        gearman_job = gear.TextJob('executor:execute', json_dumps(params),
                                    unique=uuid)
         build.__gearman_job = gearman_job
         build.__gearman_worker = None
@@ -454,7 +455,7 @@ class ExecutorClient(object):
         stop_uuid = str(uuid4().hex)
         data = dict(uuid=build.__gearman_job.unique)
         stop_job = gear.TextJob("executor:stop:%s" % build.__gearman_worker,
-                                json.dumps(data), unique=stop_uuid)
+                                json_dumps(data), unique=stop_uuid)
         self.meta_jobs[stop_uuid] = stop_job
         self.log.debug("Submitting stop job: %s", stop_job)
         self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,

zuul/lib/jsonutil.py

@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import json
+import types
+
+
+class ZuulJSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, types.MappingProxyType):
+            return dict(o)
+        return json.JSONEncoder.default(self, o)
+
+
+def json_dumps(obj):
+    return json.dumps(obj, cls=ZuulJSONEncoder)

zuul/model.py

@@ -1141,6 +1141,7 @@ class Job(ConfigObject):
     def _deepUpdate(a, b):
         # Merge nested dictionaries if possible, otherwise, overwrite
         # the value in 'a' with the value in 'b'.
+
         ret = {}
         for k, av in a.items():
             if k not in b:
@@ -1150,8 +1151,6 @@ class Job(ConfigObject):
             if (isinstance(av, (dict, types.MappingProxyType)) and
                 isinstance(bv, (dict, types.MappingProxyType))):
                 ret[k] = Job._deepUpdate(av, bv)
-            elif isinstance(bv, types.MappingProxyType):
-                ret[k] = dict(bv)
             else:
                 ret[k] = bv
         return ret