x/deb-oslo.concurrency
Code Issues Proposed changes

Merge tag '3.8.0' into debian/newton

Browse Source 
oslo.concurrency 3.8.0 release

meta:version: 3.8.0
meta:series: newton
meta:release-type: release
meta:announce: openstack-dev@lists.openstack.org
meta:pypi: yes
meta:first: yes
meta:release:Author: Joshua Harlow <jxharlow@godaddy.com>
meta:release:Commit: Doug Hellmann <doug@doughellmann.com>
meta:release:Change-Id: Ifa138a1277766d282019b46545c8b4c3488bbff8
meta:release:Code-Review+2: Doug Hellmann <doug@doughellmann.com>
meta:release:Workflow+1: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
James Page
2016-05-23 10:32:31 +01:00
parent bcf68da92a 6bae2be9ba
commit 695faf67e2
14 changed files with 189 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
doc/source/usage.rst
View File

@@ -6,11 +6,62 @@ To use oslo.concurrency in a project, import the relevant module. For
example::
from oslo_concurrency import lockutils
from oslo_concurrency import processutils
.. seealso::
* :doc:`API Documentation <api/index>`
Locking a function (local to a process)
=======================================
To ensure that a function (which is not thread safe) is only used in
a thread safe manner (typically such type of function should be refactored
to avoid this problem but if not then the following can help)::
@lockutils.synchronized('not_thread_safe')
def not_thread_safe():
pass
Once decorated later callers of this function will be able to call into
this method and the contract that two threads will **not** enter this
function at the same time will be upheld. Make sure that the names of the
locks used are carefully chosen (typically by namespacing them to your
app so that other apps will not chose the same names).
Locking a function (local to a process as well as across process)
=================================================================
To ensure that a function (which is not thread safe **or** multi-process
safe) is only used in a safe manner (typically such type of function should
be refactored to avoid this problem but if not then the following can help)::
@lockutils.synchronized('not_thread_process_safe', external=True)
def not_thread_process_safe():
pass
Once decorated later callers of this function will be able to call into
this method and the contract that two threads (or any two processes)
will **not** enter this function at the same time will be upheld. Make
sure that the names of the locks used are carefully chosen (typically by
namespacing them to your app so that other apps will not chose the same
names).
Common ways to prefix/namespace the synchronized decorator
==========================================================
Since it is **highly** recommended to prefix (or namespace) the usage
of the synchronized there are a few helpers that can make this much easier
to achieve.
An example is::
myapp_synchronized = lockutils.synchronized_with_prefix("myapp")
Then further usage of the ``lockutils.synchronized`` would instead now use
this decorator created above instead of using ``lockutils.synchronized``
directly.
Command Line Wrapper
====================

8
oslo_concurrency/locale/en_GB/LC_MESSAGES/oslo_concurrency-log-info.po
View File

@@ -5,12 +5,12 @@
#
# Translators:
# Andi Chandler <andi@gowling.com>, 2014
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
# Andreas Jaeger <jaegerandi@gmail.com>, 2016. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"Project-Id-Version: oslo.concurrency 3.6.1.dev10\n"
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
"POT-Creation-Date: 2016-04-19 12:20+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"

8
oslo_concurrency/locale/en_GB/LC_MESSAGES/oslo_concurrency.po
View File

@@ -5,12 +5,12 @@
#
# Translators:
# Andi Chandler <andi@gowling.com>, 2014-2015
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
# Andreas Jaeger <jaegerandi@gmail.com>, 2016. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"Project-Id-Version: oslo.concurrency 3.6.1.dev10\n"
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
"POT-Creation-Date: 2016-04-19 12:20+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"

8
oslo_concurrency/locale/es/LC_MESSAGES/oslo_concurrency-log-info.po
View File

@@ -5,12 +5,12 @@
#
# Translators:
# Adriana Chisco Landazábal <achisco94@gmail.com>, 2015
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
# Andreas Jaeger <jaegerandi@gmail.com>, 2016. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"Project-Id-Version: oslo.concurrency 3.6.1.dev10\n"
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
"POT-Creation-Date: 2016-04-19 12:20+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"

8
oslo_concurrency/locale/es/LC_MESSAGES/oslo_concurrency.po
View File

@@ -5,12 +5,12 @@
#
# Translators:
# Adriana Chisco Landazábal <achisco94@gmail.com>, 2015
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
# Andreas Jaeger <jaegerandi@gmail.com>, 2016. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"Project-Id-Version: oslo.concurrency 3.6.1.dev10\n"
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
"POT-Creation-Date: 2016-04-19 12:20+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"

8
oslo_concurrency/locale/fr/LC_MESSAGES/oslo_concurrency-log-info.po
View File

@@ -5,12 +5,12 @@
#
# Translators:
# Maxime COQUEREL <max.coquerel@gmail.com>, 2015
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
# Andreas Jaeger <jaegerandi@gmail.com>, 2016. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"Project-Id-Version: oslo.concurrency 3.6.1.dev10\n"
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
"POT-Creation-Date: 2016-04-19 12:20+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"

8
oslo_concurrency/locale/fr/LC_MESSAGES/oslo_concurrency.po
View File

@@ -5,12 +5,12 @@
#
# Translators:
# Maxime COQUEREL <max.coquerel@gmail.com>, 2015
# OpenStack Infra <zanata@openstack.org>, 2015. #zanata
# Andreas Jaeger <jaegerandi@gmail.com>, 2016. #zanata
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 3.2.1.dev3\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2016-01-13 08:54+0000\n"
"Project-Id-Version: oslo.concurrency 3.6.1.dev10\n"
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
"POT-Creation-Date: 2016-04-19 12:20+0000\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"

14
oslo_concurrency/locale/oslo_concurrency-log-info.pot
View File

@@ -1,24 +1,24 @@
# Translations template for oslo.concurrency.
# Copyright (C) 2015 ORGANIZATION
# Copyright (C) 2016 ORGANIZATION
# This file is distributed under the same license as the oslo.concurrency
# project.
# FIRST AUTHOR <EMAIL@ADDRESS>, 2015.
# FIRST AUTHOR <EMAIL@ADDRESS>, 2016.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 2.6.1.dev7\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2015-09-26 06:13+0000\n"
"Project-Id-Version: oslo.concurrency 3.6.1.dev10\n"
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
"POT-Creation-Date: 2016-04-21 07:17+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.1.1\n"
"Generated-By: Babel 2.2.0\n"
#: oslo_concurrency/lockutils.py:153
#: oslo_concurrency/lockutils.py:161
#, python-format
msgid "Failed to remove file %(file)s"
msgstr ""

38
oslo_concurrency/locale/oslo_concurrency.pot
View File

@@ -1,34 +1,34 @@
# Translations template for oslo.concurrency.
# Copyright (C) 2015 ORGANIZATION
# Copyright (C) 2016 ORGANIZATION
# This file is distributed under the same license as the oslo.concurrency
# project.
# FIRST AUTHOR <EMAIL@ADDRESS>, 2015.
# FIRST AUTHOR <EMAIL@ADDRESS>, 2016.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: oslo.concurrency 2.6.1.dev7\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2015-09-26 06:13+0000\n"
"Project-Id-Version: oslo.concurrency 3.6.1.dev10\n"
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/openstack-i18n/\n"
"POT-Creation-Date: 2016-04-21 07:17+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: Babel 2.1.1\n"
"Generated-By: Babel 2.2.0\n"
#: oslo_concurrency/lockutils.py:324
#: oslo_concurrency/lockutils.py:369
msgid ""
"Calling lockutils directly is no longer supported. Please use the "
"lockutils-wrapper console script instead."
msgstr ""
#: oslo_concurrency/processutils.py:72
#: oslo_concurrency/processutils.py:77
msgid "Unexpected error while running command."
msgstr ""
#: oslo_concurrency/processutils.py:75
#: oslo_concurrency/processutils.py:83
#, python-format
msgid ""
"%(description)s\n"
@@ -38,26 +38,26 @@ msgid ""
"Stderr: %(stderr)r"
msgstr ""
#: oslo_concurrency/processutils.py:221
#: oslo_concurrency/processutils.py:289
#, python-format
msgid "Got unknown keyword args: %r"
msgstr ""
#: oslo_concurrency/processutils.py:226
#: oslo_concurrency/processutils.py:294
#, python-format
msgid "Got invalid arg log_errors: %r"
msgstr ""
#: oslo_concurrency/processutils.py:232
#: oslo_concurrency/processutils.py:300
msgid "Command requested root, but did not specify a root helper."
msgstr ""
#: oslo_concurrency/processutils.py:250
#: oslo_concurrency/processutils.py:326
#, python-format
msgid "Running cmd (subprocess): %s"
msgstr ""
#: oslo_concurrency/processutils.py:312
#: oslo_concurrency/processutils.py:388
#, python-format
msgid ""
"%(desc)r\n"
@@ -67,7 +67,7 @@ msgid ""
"stderr: %(stderr)r"
msgstr ""
#: oslo_concurrency/processutils.py:321
#: oslo_concurrency/processutils.py:397
#, python-format
msgid ""
"Got an OSError\n"
@@ -75,21 +75,21 @@ msgid ""
"errno: %(errno)r"
msgstr ""
#: oslo_concurrency/processutils.py:327
#: oslo_concurrency/processutils.py:403
#, python-format
msgid "%r failed. Not Retrying."
msgstr ""
#: oslo_concurrency/processutils.py:331
#: oslo_concurrency/processutils.py:407
#, python-format
msgid "%r failed. Retrying."
msgstr ""
#: oslo_concurrency/processutils.py:379
#: oslo_concurrency/processutils.py:460
msgid "Environment not supported over SSH"
msgstr ""
#: oslo_concurrency/processutils.py:383
#: oslo_concurrency/processutils.py:464
msgid "process_input not supported over SSH"
msgstr ""

21
oslo_concurrency/prlimit.py
View File

@@ -26,8 +26,15 @@ USAGE_PROGRAM = ('%s -m oslo_concurrency.prlimit'
RESOURCES = (
# argparse argument => resource
('as', resource.RLIMIT_AS),
('core', resource.RLIMIT_CORE),
('cpu', resource.RLIMIT_CPU),
('data', resource.RLIMIT_DATA),
('fsize', resource.RLIMIT_FSIZE),
('memlock', resource.RLIMIT_MEMLOCK),
('nofile', resource.RLIMIT_NOFILE),
('nproc', resource.RLIMIT_NPROC),
('rss', resource.RLIMIT_RSS),
('stack', resource.RLIMIT_STACK),
)
@@ -35,10 +42,24 @@ def parse_args():
parser = argparse.ArgumentParser(description='prlimit', prog=USAGE_PROGRAM)
parser.add_argument('--as', type=int,
help='Address space limit in bytes')
parser.add_argument('--core', type=int,
help='Core file size limit in bytes')
parser.add_argument('--cpu', type=int,
help='CPU time limit in seconds')
parser.add_argument('--data', type=int,
help='Data size limit in bytes')
parser.add_argument('--fsize', type=int,
help='File size limit in bytes')
parser.add_argument('--memlock', type=int,
help='Locked memory limit in bytes')
parser.add_argument('--nofile', type=int,
help='Maximum number of open files')
parser.add_argument('--nproc', type=int,
help='Maximum number of processes')
parser.add_argument('--rss', type=int,
help='Maximum Resident Set Size (RSS) in bytes')
parser.add_argument('--stack', type=int,
help='Stack size limit in bytes')
parser.add_argument('program',
help='Program (absolute path)')
parser.add_argument('program_args', metavar="arg", nargs='...',

38
oslo_concurrency/processutils.py
View File

@@ -134,16 +134,36 @@ class ProcessLimits(object):
Attributes:
* address_space: Address space limit in bytes
* number_files: Maximum number of open files.
* core_file_size: Core file size limit in bytes
* cpu_time: CPU time limit in seconds
* data_size: Data size limit in bytes
* file_size: File size limit in bytes
* memory_locked: Locked memory limit in bytes
* number_files: Maximum number of open files
* number_processes: Maximum number of processes
* resident_set_size: Maximum Resident Set Size (RSS) in bytes
* stack_size: Stack size limit in bytes
This object can be used for the *prlimit* parameter of :func:`execute`.
"""
_LIMITS = {
"address_space": "--as",
"core_file_size": "--core",
"cpu_time": "--cpu",
"data_size": "--data",
"file_size": "--fsize",
"memory_locked": "--memlock",
"number_files": "--nofile",
"number_processes": "--nproc",
"resident_set_size": "--rss",
"stack_size": "--stack",
}
def __init__(self, **kw):
self.address_space = kw.pop('address_space', None)
self.number_files = kw.pop('number_files', None)
self.resident_set_size = kw.pop('resident_set_size', None)
for limit in self._LIMITS.keys():
setattr(self, limit, kw.pop(limit, None))
if kw:
raise ValueError("invalid limits: %s"
% ', '.join(sorted(kw.keys())))
@@ -151,12 +171,10 @@ class ProcessLimits(object):
def prlimit_args(self):
"""Create a list of arguments for the prlimit command line."""
args = []
if self.address_space:
args.append('--as=%s' % self.address_space)
if self.number_files:
args.append('--nofile=%s' % self.number_files)
if self.resident_set_size:
args.append('--rss=%s' % self.resident_set_size)
for limit in self._LIMITS.keys():
val = getattr(self, limit)
if val is not None:
args.append("%s=%s" % (self._LIMITS[limit], val))
return args

37
oslo_concurrency/tests/unit/test_processutils.py
View File

@@ -752,7 +752,7 @@ class PrlimitTestCase(test_base.BaseTestCase):
# Create a new soft limit for a resource, lower than the current
# soft limit.
soft_limit, hard_limit = resource.getrlimit(res)
if soft_limit < 0:
if soft_limit <= 0:
soft_limit = default_limit
else:
soft_limit -= substract
@@ -790,6 +790,31 @@ class PrlimitTestCase(test_base.BaseTestCase):
prlimit = self.limit_address_space()
self.check_limit(prlimit, 'RLIMIT_AS', prlimit.address_space)
def test_core_size(self):
size = self.soft_limit(resource.RLIMIT_CORE, 1, 1024)
prlimit = processutils.ProcessLimits(core_file_size=size)
self.check_limit(prlimit, 'RLIMIT_CORE', prlimit.core_file_size)
def test_cpu_time(self):
time = self.soft_limit(resource.RLIMIT_CPU, 1, 1024)
prlimit = processutils.ProcessLimits(cpu_time=time)
self.check_limit(prlimit, 'RLIMIT_CPU', prlimit.cpu_time)
def test_data_size(self):
max_memory = self.memory_limit(resource.RLIMIT_DATA)
prlimit = processutils.ProcessLimits(data_size=max_memory)
self.check_limit(prlimit, 'RLIMIT_DATA', max_memory)
def test_file_size(self):
size = self.soft_limit(resource.RLIMIT_FSIZE, 1, 1024)
prlimit = processutils.ProcessLimits(file_size=size)
self.check_limit(prlimit, 'RLIMIT_FSIZE', prlimit.file_size)
def test_memory_locked(self):
max_memory = self.memory_limit(resource.RLIMIT_MEMLOCK)
prlimit = processutils.ProcessLimits(memory_locked=max_memory)
self.check_limit(prlimit, 'RLIMIT_MEMLOCK', max_memory)
def test_resident_set_size(self):
max_memory = self.memory_limit(resource.RLIMIT_RSS)
prlimit = processutils.ProcessLimits(resident_set_size=max_memory)
@@ -800,6 +825,16 @@ class PrlimitTestCase(test_base.BaseTestCase):
prlimit = processutils.ProcessLimits(number_files=nfiles)
self.check_limit(prlimit, 'RLIMIT_NOFILE', nfiles)
def test_number_processes(self):
nprocs = self.soft_limit(resource.RLIMIT_NPROC, 1, 65535)
prlimit = processutils.ProcessLimits(number_processes=nprocs)
self.check_limit(prlimit, 'RLIMIT_NPROC', nprocs)
def test_stack_size(self):
max_memory = self.memory_limit(resource.RLIMIT_STACK)
prlimit = processutils.ProcessLimits(stack_size=max_memory)
self.check_limit(prlimit, 'RLIMIT_STACK', max_memory)
def test_unsupported_prlimit(self):
self.assertRaises(ValueError, processutils.ProcessLimits, xxx=33)

5
requirements.txt
View File

@@ -3,10 +3,9 @@
# process, which may cause wedges in the gate later.
pbr>=1.6 # Apache-2.0
Babel>=1.3 # BSD
enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
iso8601>=0.1.9 # MIT
oslo.config>=3.7.0 # Apache-2.0
iso8601>=0.1.11 # MIT
oslo.config>=3.9.0 # Apache-2.0
oslo.i18n>=2.1.0 # Apache-2.0
oslo.utils>=3.5.0 # Apache-2.0
six>=1.9.0 # MIT

2
test-requirements.txt
View File

@@ -6,7 +6,7 @@ hacking<0.11,>=0.10.0
oslotest>=1.10.0 # Apache-2.0
coverage>=3.6 # Apache-2.0
futures>=3.0;python_version=='2.7' or python_version=='2.6' # BSD
fixtures>=1.3.1 # Apache-2.0/BSD
fixtures<2.0,>=1.3.1 # Apache-2.0/BSD
# These are needed for docs generation
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0