Browse Source

Merge "Fix serialization of structures that might contain YAQL types"

tags/10.0.0.0b3
Zuul 3 months ago
committed by Gerrit Code Review
parent
commit
869cac9def
8 changed files with 161 additions and 45 deletions
  1. +3
    -3
      mistral/actions/std_actions.py
  2. +4
    -15
      mistral/db/sqlalchemy/types.py
  3. +4
    -1
      mistral/expressions/yaql_expression.py
  4. +3
    -7
      mistral/tests/unit/engine/test_dataflow.py
  5. +2
    -0
      mistral/tests/unit/expressions/test_yaql_expression.py
  6. +89
    -0
      mistral/tests/unit/expressions/test_yaql_json_serialization.py
  7. +51
    -0
      mistral/utils/__init__.py
  8. +5
    -19
      mistral/utils/javascript.py

+ 3
- 3
mistral/actions/std_actions.py View File

@@ -16,7 +16,6 @@
from email import header
from email.mime import multipart
from email.mime import text
import json as json_lib
import smtplib
import time

@@ -25,6 +24,7 @@ import requests
import six

from mistral import exceptions as exc
from mistral import utils
from mistral.utils import javascript
from mistral.utils import ssh_utils
from mistral_lib import actions
@@ -182,7 +182,7 @@ class HTTPAction(actions.Action):
self.url = url
self.method = method
self.params = params
self.body = json_lib.dumps(body) if isinstance(body, dict) else body
self.body = utils.to_json_str(body) if isinstance(body, dict) else body
self.json = json
self.headers = headers
self.cookies = cookies
@@ -456,7 +456,7 @@ class SSHAction(actions.Action):
return raise_exc(parent_exc=e)

def test(self, context):
return json_lib.dumps(self.params)
return utils.to_json_str(self.params)


class SSHProxiedAction(SSHAction):


+ 4
- 15
mistral/db/sqlalchemy/types.py View File

@@ -16,11 +16,12 @@
# expressed by json-strings
#

from oslo_serialization import jsonutils
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from sqlalchemy.ext import mutable

from mistral import utils


class JsonEncoded(sa.TypeDecorator):
"""Represents an immutable structure as a json-encoded string."""
@@ -28,22 +29,10 @@ class JsonEncoded(sa.TypeDecorator):
impl = sa.Text

def process_bind_param(self, value, dialect):
if value is not None:
# We need to convert the root of the given object graph into
# a primitive by hand so that we also enable conversion of
# object of custom classes into primitives. Otherwise, they are
# ignored by the "json" lib.
value = jsonutils.dumps(
jsonutils.to_primitive(value, convert_instances=True)
)

return value
return utils.to_json_str(value)

def process_result_value(self, value, dialect):
if value is not None:
value = jsonutils.loads(value)

return value
return utils.from_json_str(value)


class MutableList(mutable.Mutable, list):


+ 4
- 1
mistral/expressions/yaql_expression.py View File

@@ -135,7 +135,10 @@ def _sanitize_yaql_result(result):
if isinstance(result, yaql_utils.FrozenDict):
return result._d

return result if not inspect.isgenerator(result) else list(result)
if inspect.isgenerator(result):
return list(result)

return result


class YAQLEvaluator(base.Evaluator):


+ 3
- 7
mistral/tests/unit/engine/test_dataflow.py View File

@@ -14,7 +14,6 @@
# limitations under the License.

from oslo_config import cfg
from oslo_serialization import jsonutils

from mistral.db.v2 import api as db_api
from mistral.db.v2.sqlalchemy import models
@@ -24,6 +23,7 @@ from mistral.services import workbooks as wb_service
from mistral.services import workflows as wf_service
from mistral.tests.unit import base as test_base
from mistral.tests.unit.engine import base as engine_test_base
from mistral import utils
from mistral.workflow import data_flow
from mistral.workflow import states

@@ -1444,9 +1444,7 @@ class DataFlowTest(test_base.BaseTest):
{'k2': 'v2'},
)

json_str = jsonutils.dumps(
jsonutils.to_primitive(ctx, convert_instances=True)
)
json_str = utils.to_json_str(ctx)

self.assertIsNotNone(json_str)
self.assertNotEqual('{}', json_str)
@@ -1464,9 +1462,7 @@ class DataFlowTest(test_base.BaseTest):

d = {'root': ctx}

json_str = jsonutils.dumps(
jsonutils.to_primitive(d, convert_instances=True)
)
json_str = utils.to_json_str(d)

self.assertIsNotNone(json_str)
self.assertNotEqual('{"root": {}}', json_str)


+ 2
- 0
mistral/tests/unit/expressions/test_yaql_expression.py View File

@@ -27,6 +27,7 @@ from mistral.expressions import yaql_expression as expr
from mistral.tests.unit import base
from mistral_lib import utils


CONF = cfg.CONF

DATA = {
@@ -96,6 +97,7 @@ class YaqlEvaluatorTest(base.BaseTest):
'$.servers.where($.name = ubuntu)',
SERVERS
)

item = list(res)[0]

self.assertEqual({'name': 'ubuntu'}, item)


+ 89
- 0
mistral/tests/unit/expressions/test_yaql_json_serialization.py View File

@@ -0,0 +1,89 @@
# Copyright 2013 - Mirantis, Inc.
# Copyright 2015 - StackStorm, Inc.
# Copyright 2016 - Brocade Communications Systems, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from yaql.language import utils as yaql_utils

from mistral.tests.unit import base
from mistral import utils


class YaqlJsonSerializationTest(base.BaseTest):
def test_serialize_frozen_dict(self):
data = yaql_utils.FrozenDict(a=1, b=2, c=iter([1, 2, 3]))

json_str = utils.to_json_str(data)

self.assertIsNotNone(json_str)

self.assertIn('"a": 1', json_str)
self.assertIn('"b": 2', json_str)
self.assertIn('"c": [1, 2, 3]', json_str)

def test_serialize_generator(self):
def _list_stream(_list):
for i in _list:
yield i

gen = _list_stream(
[1, yaql_utils.FrozenDict(a=1), _list_stream([12, 15])]
)

self.assertEqual('[1, {"a": 1}, [12, 15]]', utils.to_json_str(gen))

def test_serialize_dict_of_generators(self):
def _f(cnt):
for i in range(1, cnt + 1):
yield i

data = {'numbers': _f(3)}

self.assertEqual('{"numbers": [1, 2, 3]}', utils.to_json_str(data))

def test_serialize_range(self):
self.assertEqual("[1, 2, 3, 4]", utils.to_json_str(range(1, 5)))

def test_serialize_iterator_of_frozen_dicts(self):
data = iter(
[
yaql_utils.FrozenDict(a=1, b=2, c=iter([1, 2, 3])),
yaql_utils.FrozenDict(
a=11,
b=yaql_utils.FrozenDict(b='222'),
c=iter(
[
1,
yaql_utils.FrozenDict(
a=iter([4, yaql_utils.FrozenDict(a=99)])
)
]
)
)
]
)

json_str = utils.to_json_str(data)

self.assertIsNotNone(json_str)

# Checking the first item.
self.assertIn('"a": 1', json_str)
self.assertIn('"b": 2', json_str)
self.assertIn('"c": [1, 2, 3]', json_str)

# Checking the first item.
self.assertIn('"a": 11', json_str)
self.assertIn('"b": {"b": "222"}', json_str)
self.assertIn('"c": [1, {"a": [4, {"a": 99}]}]', json_str)

+ 51
- 0
mistral/utils/__init__.py View File

@@ -15,12 +15,14 @@
# limitations under the License.

import contextlib
import inspect
import os
import shutil
import tempfile
import threading

from oslo_concurrency import processutils
from oslo_serialization import jsonutils

from mistral import exceptions as exc

@@ -101,3 +103,52 @@ def generate_key_pair(key_length=2048):
public_key = open(public_key_path).read()

return private_key, public_key


def to_json_str(obj):
"""Serializes an object into a JSON string.

:param obj: Object to serialize.
:return: JSON string.
"""

if obj is None:
return None

def _fallback(value):
if inspect.isgenerator(value):
result = list(value)

# The result of the generator call may be again not primitive
# so we need to call "to_primitive" again with the same fallback
# function. Note that the endless recursion here is not a problem
# because "to_primitive" limits the depth for custom classes,
# if they are present in the object graph being traversed.
return jsonutils.to_primitive(
result,
convert_instances=True,
fallback=_fallback
)

return value

# We need to convert the root of the given object graph into
# a primitive by hand so that we also enable conversion of
# object of custom classes into primitives. Otherwise, they are
# ignored by the "json" lib.
return jsonutils.dumps(
jsonutils.to_primitive(obj, convert_instances=True, fallback=_fallback)
)


def from_json_str(json_str):
"""Reconstructs an object from a JSON string.

:param json_str: A JSON string.
:return: Deserialized object.
"""

if json_str is None:
return None

return jsonutils.loads(json_str)

+ 5
- 19
mistral/utils/javascript.py View File

@@ -17,8 +17,8 @@ import abc

from mistral import config as cfg
from mistral import exceptions as exc
from mistral import utils

from oslo_serialization import jsonutils
from oslo_utils import importutils
from stevedore import driver
from stevedore import extension
@@ -55,13 +55,7 @@ class PyV8Evaluator(JSEvaluator):

with _PYV8.JSContext() as js_ctx:
# Prepare data context and way for interaction with it.
# NOTE: it's important to enable conversion of custom types
# into JSON to account for classes like ContextView.
ctx_str = jsonutils.dumps(
jsonutils.to_primitive(ctx, convert_instances=True)
)

js_ctx.eval('$ = %s' % ctx_str)
js_ctx.eval('$ = %s' % utils.to_json_str(ctx))

result = js_ctx.eval(script)

@@ -78,11 +72,7 @@ class V8EvalEvaluator(JSEvaluator):

v8 = _V8EVAL.V8()

# NOTE: it's important to enable conversion of custom types
# into JSON to account for classes like ContextView.
ctx_str = jsonutils.dumps(
jsonutils.to_primitive(ctx, convert_instances=True)
)
ctx_str = utils.to_json_str(ctx)

return v8.eval(
('$ = %s; %s' % (ctx_str, script)).encode(encoding='UTF-8')
@@ -100,14 +90,10 @@ class PyMiniRacerEvaluator(JSEvaluator):

js_ctx = _PY_MINI_RACER.MiniRacer()

# NOTE: it's important to enable conversion of custom types
# into JSON to account for classes like ContextView.
ctx_str = jsonutils.dumps(
jsonutils.to_primitive(ctx, convert_instances=True)
return js_ctx.eval(
'$ = {}; {}'.format(utils.to_json_str(ctx), script)
)

return js_ctx.eval(('$ = {}; {}'.format(ctx_str, script)))


_mgr = extension.ExtensionManager(
namespace='mistral.expression.evaluators',


Loading…
Cancel
Save