From 88fdfde8e27335a774cb5eafdaa1760d485d85dd Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Tue, 28 Mar 2017 16:22:42 -0700 Subject: [PATCH] Use libyaml if possible PyYAML doesn't automatically use the much faster and more memory efficient libyaml bindings, even if the extension is available. So we provide our own module that exports the pieces needed to use the faster one, or fall back to the pure python implementation. Change-Id: I7ee99f5017cb83153ab8fa9bc23548ed639777c1 --- tests/make_playbooks.py | 2 +- tests/unit/test_model.py | 2 +- zuul/cmd/__init__.py | 2 +- zuul/configloader.py | 2 +- zuul/executor/ansiblelaunchserver.py | 2 +- zuul/executor/server.py | 2 +- zuul/lib/cloner.py | 2 +- zuul/lib/yamlutil.py | 32 ++++++++++++++++++++++++++++ 8 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 zuul/lib/yamlutil.py diff --git a/tests/make_playbooks.py b/tests/make_playbooks.py index 17acba8b25..93c37bc815 100755 --- a/tests/make_playbooks.py +++ b/tests/make_playbooks.py @@ -14,7 +14,7 @@ import os -import yaml +from zuul.lib import yamlutil as yaml FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'fixtures') diff --git a/tests/unit/test_model.py b/tests/unit/test_model.py index f906095b27..2167a3b713 100644 --- a/tests/unit/test_model.py +++ b/tests/unit/test_model.py @@ -18,11 +18,11 @@ import random import fixtures import testtools -import yaml from zuul import model from zuul import configloader from zuul.lib import encryption +from zuul.lib import yamlutil as yaml from tests.base import BaseTestCase, FIXTURE_DIR diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py index 9fa4c03e99..f2a2612acd 100644 --- a/zuul/cmd/__init__.py +++ b/zuul/cmd/__init__.py @@ -24,10 +24,10 @@ import signal import sys import traceback -import yaml yappi = extras.try_import('yappi') import zuul.lib.connections +from zuul.lib import yamlutil as yaml # Do not import modules that will pull in paramiko which must not be # imported until after the daemonization. diff --git a/zuul/configloader.py b/zuul/configloader.py index 64c8db4f4f..c54946158d 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -15,13 +15,13 @@ import copy import os import logging import six -import yaml import pprint import textwrap import voluptuous as vs from zuul import model +from zuul.lib import yamlutil as yaml import zuul.manager.dependent import zuul.manager.independent from zuul import change_matcher diff --git a/zuul/executor/ansiblelaunchserver.py b/zuul/executor/ansiblelaunchserver.py index 875cf2be24..0202bdda9b 100644 --- a/zuul/executor/ansiblelaunchserver.py +++ b/zuul/executor/ansiblelaunchserver.py @@ -35,13 +35,13 @@ import uuid import Queue import gear -import yaml import jenkins_jobs.builder import jenkins_jobs.formatter import zmq import zuul.ansible.library from zuul.lib import commandsocket +from zuul.lib import yamlutil as yaml ANSIBLE_WATCHDOG_GRACE = 5 * 60 ANSIBLE_DEFAULT_TIMEOUT = 2 * 60 * 60 diff --git a/zuul/executor/server.py b/zuul/executor/server.py index d0741bb503..c397dcac57 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -24,7 +24,7 @@ import tempfile import threading import time import traceback -import yaml +from zuul.lib.yamlutil import yaml import gear import git diff --git a/zuul/lib/cloner.py b/zuul/lib/cloner.py index 18dea91193..bec8ebee49 100644 --- a/zuul/lib/cloner.py +++ b/zuul/lib/cloner.py @@ -17,13 +17,13 @@ import git import logging import os import re -import yaml import six from git import GitCommandError from zuul import exceptions from zuul.lib.clonemapper import CloneMapper +from zuul.lib import yamlutil as yaml from zuul.merger.merger import Repo diff --git a/zuul/lib/yamlutil.py b/zuul/lib/yamlutil.py new file mode 100644 index 0000000000..2419906cdc --- /dev/null +++ b/zuul/lib/yamlutil.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import yaml +from yaml import YAMLObject, YAMLError # noqa: F401 + +try: + from yaml import cyaml + import _yaml + SafeLoader = cyaml.CSafeLoader + SafeDumper = cyaml.CSafeDumper + Mark = _yaml.Mark +except ImportError: + SafeLoader = yaml.SafeLoader + SafeDumper = yaml.SafeDumper + Mark = yaml.Mark + + +def safe_load(stream, *args, **kwargs): + return yaml.load(stream, *args, Loader=SafeLoader, **kwargs) + + +def safe_dump(stream, *args, **kwargs): + return yaml.dump(stream, *args, Dumper=SafeDumper, **kwargs)