Merge tag '3.6.0' into debian/mitaka

oslo.concurrency 3.6.0 release

meta:version: 3.6.0
meta:series: mitaka
meta:release-type: release
meta:announce: openstack-dev@lists.openstack.org
meta:pypi: yes
This commit is contained in:
Corey Bryant
2016-03-01 09:27:00 -05:00
15 changed files with 346 additions and 70 deletions

View File

@@ -16,7 +16,7 @@
import oslo_i18n
_translators = oslo_i18n.TranslatorFactory(domain='oslo.concurrency')
_translators = oslo_i18n.TranslatorFactory(domain='oslo_concurrency')
# The primary translation function using the well-known name "_"
_ = _translators.primary

View File

@@ -8,19 +8,19 @@
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 2.8.1.dev1\n"
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2015-10-23 06:15+0000\n"
"PO-Revision-Date: 2015-06-10 11:06+0000\n"
"Last-Translator: openstackjenkins <jenkins@openstack.org>\n"
"Language-Team: English (United Kingdom)\n"
"Language: en-GB\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"PO-Revision-Date: 2015-06-10 11:06+0000\n"
"Last-Translator: openstackjenkins <jenkins@openstack.org>\n"
"Language: en-GB\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"Generated-By: Babel 2.0\n"
"X-Generator: Zanata 3.7.1\n"
"X-Generator: Zanata 3.7.3\n"
"Language-Team: English (United Kingdom)\n"
#, python-format
msgid "Failed to remove file %(file)s"

View File

@@ -1,4 +1,4 @@
# English (United Kingdom) translations for oslo.concurrency.
# Translations template for oslo.concurrency.
# Copyright (C) 2015 ORGANIZATION
# This file is distributed under the same license as the oslo.concurrency
# project.
@@ -8,18 +8,19 @@
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 2.8.1.dev1\n"
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2015-10-23 06:15+0000\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"PO-Revision-Date: 2015-06-10 11:06+0000\n"
"Last-Translator: openstackjenkins <jenkins@openstack.org>\n"
"Language: en_GB\n"
"Language: en-GB\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"Generated-By: Babel 2.0\n"
"X-Generator: Zanata 3.7.3\n"
"Language-Team: English (United Kingdom)\n"
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.1.1\n"
#, python-format
msgid ""

View File

@@ -8,19 +8,19 @@
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 2.8.1.dev1\n"
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2015-10-23 06:15+0000\n"
"PO-Revision-Date: 2015-06-22 09:27+0000\n"
"Last-Translator: Adriana Chisco Landazábal <achisco94@gmail.com>\n"
"Language-Team: Spanish\n"
"Language: es\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"PO-Revision-Date: 2015-06-22 09:27+0000\n"
"Last-Translator: Adriana Chisco Landazábal <achisco94@gmail.com>\n"
"Language: es\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"Generated-By: Babel 2.0\n"
"X-Generator: Zanata 3.7.1\n"
"X-Generator: Zanata 3.7.3\n"
"Language-Team: Spanish\n"
#, python-format
msgid "Failed to remove file %(file)s"

View File

@@ -1,4 +1,4 @@
# Spanish translations for oslo.concurrency.
# Translations template for oslo.concurrency.
# Copyright (C) 2015 ORGANIZATION
# This file is distributed under the same license as the oslo.concurrency
# project.
@@ -8,18 +8,19 @@
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 2.8.1.dev1\n"
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2015-10-23 06:15+0000\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"PO-Revision-Date: 2015-06-22 09:35+0000\n"
"Last-Translator: Adriana Chisco Landazábal <achisco94@gmail.com>\n"
"Language: es\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"Generated-By: Babel 2.0\n"
"X-Generator: Zanata 3.7.3\n"
"Language-Team: Spanish\n"
"Plural-Forms: nplurals=2; plural=(n != 1)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.1.1\n"
#, python-format
msgid ""

View File

@@ -8,19 +8,19 @@
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 2.8.1.dev1\n"
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2015-10-23 06:15+0000\n"
"PO-Revision-Date: 2015-06-10 11:06+0000\n"
"Last-Translator: openstackjenkins <jenkins@openstack.org>\n"
"Language-Team: French\n"
"Language: fr\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"PO-Revision-Date: 2015-06-10 11:06+0000\n"
"Last-Translator: openstackjenkins <jenkins@openstack.org>\n"
"Language: fr\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"
"Generated-By: Babel 2.0\n"
"X-Generator: Zanata 3.7.1\n"
"X-Generator: Zanata 3.7.3\n"
"Language-Team: French\n"
#, python-format
msgid "Failed to remove file %(file)s"

View File

@@ -1,4 +1,4 @@
# French translations for oslo.concurrency.
# Translations template for oslo.concurrency.
# Copyright (C) 2015 ORGANIZATION
# This file is distributed under the same license as the oslo.concurrency
# project.
@@ -8,18 +8,19 @@
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 2.8.1.dev1\n"
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2015-10-23 06:15+0000\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"PO-Revision-Date: 2015-06-10 11:06+0000\n"
"Last-Translator: openstackjenkins <jenkins@openstack.org>\n"
"Language: fr\n"
"Plural-Forms: nplurals=2; plural=(n > 1);\n"
"Generated-By: Babel 2.0\n"
"X-Generator: Zanata 3.7.3\n"
"Language-Team: French\n"
"Plural-Forms: nplurals=2; plural=(n > 1)\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.1.1\n"
#, python-format
msgid ""

View File

@@ -0,0 +1,89 @@
# Copyright 2016 Red Hat.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from __future__ import print_function
import argparse
import os
import resource
import sys
USAGE_PROGRAM = ('%s -m oslo_concurrency.prlimit'
% os.path.basename(sys.executable))
RESOURCES = (
# argparse argument => resource
('as', resource.RLIMIT_AS),
('nofile', resource.RLIMIT_NOFILE),
('rss', resource.RLIMIT_RSS),
)
def parse_args():
parser = argparse.ArgumentParser(description='prlimit', prog=USAGE_PROGRAM)
parser.add_argument('--as', type=int,
help='Address space limit in bytes')
parser.add_argument('--nofile', type=int,
help='Maximum number of open files')
parser.add_argument('--rss', type=int,
help='Maximum Resident Set Size (RSS) in bytes')
parser.add_argument('program',
help='Program (absolute path)')
parser.add_argument('program_args', metavar="arg", nargs='...',
help='Program parameters')
args = parser.parse_args()
return args
def main():
args = parse_args()
program = args.program
if not os.path.isabs(program):
# program uses a relative path: try to find the absolute path
# to the executable
if sys.version_info >= (3, 3):
import shutil
program_abs = shutil.which(program)
else:
import distutils.spawn
program_abs = distutils.spawn.find_executable(program)
if program_abs:
program = program_abs
for arg_name, rlimit in RESOURCES:
value = getattr(args, arg_name)
if value is None:
continue
try:
resource.setrlimit(rlimit, (value, value))
except ValueError as exc:
print("%s: failed to set the %s resource limit: %s"
% (USAGE_PROGRAM, arg_name.upper(), exc),
file=sys.stderr)
sys.exit(1)
try:
os.execv(program, [program] + args.program_args)
except Exception as exc:
print("%s: failed to execute %s: %s"
% (USAGE_PROGRAM, program, exc),
file=sys.stderr)
sys.exit(1)
if __name__ == "__main__":
main()

View File

@@ -24,6 +24,7 @@ import os
import random
import shlex
import signal
import sys
import time
import enum
@@ -62,26 +63,33 @@ class UnknownArgumentError(Exception):
class ProcessExecutionError(Exception):
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
description=None):
super(ProcessExecutionError, self).__init__(
stdout, stderr, exit_code, cmd, description)
self.exit_code = exit_code
self.stderr = stderr
self.stdout = stdout
self.cmd = cmd
self.description = description
def __str__(self):
description = self.description
if description is None:
description = _("Unexpected error while running command.")
exit_code = self.exit_code
if exit_code is None:
exit_code = '-'
message = _('%(description)s\n'
'Command: %(cmd)s\n'
'Exit code: %(exit_code)s\n'
'Stdout: %(stdout)r\n'
'Stderr: %(stderr)r') % {'description': description,
'cmd': cmd,
'cmd': self.cmd,
'exit_code': exit_code,
'stdout': stdout,
'stderr': stderr}
super(ProcessExecutionError, self).__init__(message)
'stdout': self.stdout,
'stderr': self.stderr}
return message
class NoRootWrapSpecified(Exception):
@@ -120,6 +128,38 @@ LOG_FINAL_ERROR = LogErrors.FINAL
LOG_DEFAULT_ERROR = LogErrors.DEFAULT
class ProcessLimits(object):
"""Resource limits on a process.
Attributes:
* address_space: Address space limit in bytes
* number_files: Maximum number of open files.
* resident_set_size: Maximum Resident Set Size (RSS) in bytes
This object can be used for the *prlimit* parameter of :func:`execute`.
"""
def __init__(self, **kw):
self.address_space = kw.pop('address_space', None)
self.number_files = kw.pop('number_files', None)
self.resident_set_size = kw.pop('resident_set_size', None)
if kw:
raise ValueError("invalid limits: %s"
% ', '.join(sorted(kw.keys())))
def prlimit_args(self):
"""Create a list of arguments for the prlimit command line."""
args = []
if self.address_space:
args.append('--as=%s' % self.address_space)
if self.number_files:
args.append('--nofile=%s' % self.number_files)
if self.resident_set_size:
args.append('--rss=%s' % self.resident_set_size)
return args
def execute(*cmd, **kwargs):
"""Helper method to shell out and execute a command through subprocess.
@@ -188,12 +228,22 @@ def execute(*cmd, **kwargs):
subprocess.Popen on windows (throws a
ValueError)
:type preexec_fn: function()
:param prlimit: Set resource limits on the child process. See
below for a detailed description.
:type prlimit: :class:`ProcessLimits`
:returns: (stdout, stderr) from process execution
:raises: :class:`UnknownArgumentError` on
receiving unknown arguments
:raises: :class:`ProcessExecutionError`
:raises: :class:`OSError`
The *prlimit* parameter can be used to set resource limits on the child
process. If this parameter is used, the child process will be spawned by a
wrapper process which will set limits before spawning the command.
.. versionchanged:: 3.4
Added *prlimit* optional parameter.
.. versionchanged:: 1.5
Added *cwd* optional parameter.
@@ -227,6 +277,7 @@ def execute(*cmd, **kwargs):
on_execute = kwargs.pop('on_execute', None)
on_completion = kwargs.pop('on_completion', None)
preexec_fn = kwargs.pop('preexec_fn', None)
prlimit = kwargs.pop('prlimit', None)
if isinstance(check_exit_code, bool):
ignore_exit_code = not check_exit_code
@@ -256,6 +307,14 @@ def execute(*cmd, **kwargs):
cmd = shlex.split(root_helper) + list(cmd)
cmd = [str(c) for c in cmd]
if prlimit:
args = [sys.executable, '-m', 'oslo_concurrency.prlimit']
args.extend(prlimit.prlimit_args())
args.append('--')
args.extend(cmd)
cmd = args
sanitized_cmd = strutils.mask_password(' '.join(cmd))
watch = timeutils.StopWatch()

View File

@@ -19,6 +19,8 @@ import errno
import logging
import multiprocessing
import os
import pickle
import resource
import stat
import subprocess
import sys
@@ -466,6 +468,21 @@ grep foo
def test_binary_undecodable_bytes_error(self):
self.check_undecodable_bytes_error(True)
def test_picklable(self):
exc = processutils.ProcessExecutionError(
stdout='my stdout', stderr='my stderr',
exit_code=42, cmd='my cmd',
description='my description')
exc_message = str(exc)
exc = pickle.loads(pickle.dumps(exc))
self.assertEqual('my stdout', exc.stdout)
self.assertEqual('my stderr', exc.stderr)
self.assertEqual(42, exc.exit_code)
self.assertEqual('my cmd', exc.cmd)
self.assertEqual('my description', exc.description)
self.assertEqual(exc_message, str(exc))
class ProcessExecutionErrorLoggingTest(test_base.BaseTestCase):
def setUp(self):
@@ -724,3 +741,111 @@ class SshExecuteTestCase(test_base.BaseTestCase):
def test_compromising_ssh6(self):
self._test_compromising_ssh(rc=-1, check=False)
class PrlimitTestCase(test_base.BaseTestCase):
# Simply program that does nothing and returns an exit code 0.
# Use Python to be portable.
SIMPLE_PROGRAM = [sys.executable, '-c', 'pass']
def soft_limit(self, res, substract, default_limit):
# Create a new soft limit for a resource, lower than the current
# soft limit.
soft_limit, hard_limit = resource.getrlimit(res)
if soft_limit < 0:
soft_limit = default_limit
else:
soft_limit -= substract
return soft_limit
def memory_limit(self, res):
# Substract 1 kB just to get a different limit. Don't substract too
# much to avoid memory allocation issues.
#
# Use 1 GB by default. Limit high enough to be able to load shared
# libraries. Limit low enough to be work on 32-bit platforms.
return self.soft_limit(res, 1024, 1024 ** 3)
def limit_address_space(self):
max_memory = self.memory_limit(resource.RLIMIT_AS)
return processutils.ProcessLimits(address_space=max_memory)
def test_simple(self):
# Simple test running a program (/bin/true) with no parameter
prlimit = self.limit_address_space()
stdout, stderr = processutils.execute(*self.SIMPLE_PROGRAM,
prlimit=prlimit)
self.assertEqual(stdout.rstrip(), '')
self.assertEqual(stderr.rstrip(), '')
def check_limit(self, prlimit, resource, value):
code = ';'.join(('import resource',
'print(resource.getrlimit(resource.%s))' % resource))
args = [sys.executable, '-c', code]
stdout, stderr = processutils.execute(*args, prlimit=prlimit)
expected = (value, value)
self.assertEqual(stdout.rstrip(), str(expected))
def test_address_space(self):
prlimit = self.limit_address_space()
self.check_limit(prlimit, 'RLIMIT_AS', prlimit.address_space)
def test_resident_set_size(self):
max_memory = self.memory_limit(resource.RLIMIT_RSS)
prlimit = processutils.ProcessLimits(resident_set_size=max_memory)
self.check_limit(prlimit, 'RLIMIT_RSS', max_memory)
def test_number_files(self):
nfiles = self.soft_limit(resource.RLIMIT_NOFILE, 1, 1024)
prlimit = processutils.ProcessLimits(number_files=nfiles)
self.check_limit(prlimit, 'RLIMIT_NOFILE', nfiles)
def test_unsupported_prlimit(self):
self.assertRaises(ValueError, processutils.ProcessLimits, xxx=33)
def test_relative_path(self):
prlimit = self.limit_address_space()
program = sys.executable
env = dict(os.environ)
env['PATH'] = os.path.dirname(program)
args = [os.path.basename(program), '-c', 'pass']
processutils.execute(*args, prlimit=prlimit, env_variables=env)
def test_execv_error(self):
prlimit = self.limit_address_space()
args = ['/missing_path/dont_exist/program']
try:
processutils.execute(*args, prlimit=prlimit)
except processutils.ProcessExecutionError as exc:
self.assertEqual(exc.exit_code, 1)
self.assertEqual(exc.stdout, '')
expected = ('%s -m oslo_concurrency.prlimit: '
'failed to execute /missing_path/dont_exist/program: '
% os.path.basename(sys.executable))
self.assertIn(expected, exc.stderr)
else:
self.fail("ProcessExecutionError not raised")
def test_setrlimit_error(self):
prlimit = self.limit_address_space()
# trying to set a limit higher than the current hard limit
# with setrlimit() should fail.
higher_limit = prlimit.address_space + 1024
args = [sys.executable, '-m', 'oslo_concurrency.prlimit',
'--as=%s' % higher_limit,
'--']
args.extend(self.SIMPLE_PROGRAM)
try:
processutils.execute(*args, prlimit=prlimit)
except processutils.ProcessExecutionError as exc:
self.assertEqual(exc.exit_code, 1)
self.assertEqual(exc.stdout, '')
expected = ('%s -m oslo_concurrency.prlimit: '
'failed to set the AS resource limit: '
% os.path.basename(sys.executable))
self.assertIn(expected, exc.stderr)
else:
self.fail("ProcessExecutionError not raised")

View File

@@ -2,13 +2,13 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
pbr>=1.6
Babel>=1.3
enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3'
iso8601>=0.1.9
oslo.config>=3.2.0 # Apache-2.0
oslo.i18n>=1.5.0 # Apache-2.0
oslo.utils>=3.2.0 # Apache-2.0
six>=1.9.0
pbr>=1.6 # Apache-2.0
Babel>=1.3 # BSD
enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
iso8601>=0.1.9 # MIT
oslo.config>=3.4.0 # Apache-2.0
oslo.i18n>=2.1.0 # Apache-2.0
oslo.utils>=3.5.0 # Apache-2.0
six>=1.9.0 # MIT
fasteners>=0.7 # Apache-2.0
retrying!=1.3.0,>=1.2.3 # Apache-2.0

View File

@@ -37,18 +37,18 @@ all_files = 1
upload-dir = doc/build/html
[compile_catalog]
directory = oslo.concurrency/locale
domain = oslo.concurrency
directory = oslo_concurrency/locale
domain = oslo_concurrency
[update_catalog]
domain = oslo.concurrency
output_dir = oslo.concurrency/locale
input_file = oslo.concurrency/locale/oslo.concurrency.pot
domain = oslo_concurrency
output_dir = oslo_concurrency/locale
input_file = oslo_concurrency/locale/oslo_concurrency.pot
[extract_messages]
keywords = _ gettext ngettext l_ lazy_gettext
mapping_file = babel.cfg
output_file = oslo.concurrency/locale/oslo.concurrency.pot
output_file = oslo_concurrency/locale/oslo_concurrency.pot
[pbr]
warnerrors = True

View File

@@ -4,12 +4,12 @@
hacking<0.11,>=0.10.0
oslotest>=1.10.0 # Apache-2.0
coverage>=3.6
futures>=3.0;python_version=='2.7' or python_version=='2.6'
fixtures>=1.3.1
coverage>=3.6 # Apache-2.0
futures>=3.0;python_version=='2.7' or python_version=='2.6' # BSD
fixtures>=1.3.1 # Apache-2.0/BSD
# These are needed for docs generation
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
eventlet>=0.17.4
eventlet!=0.18.3,>=0.18.2 # MIT