Limit resource consumption of function

Usages of following resources are limited(for at least python runtime):
- cpu
- process number
- file descriptor
- file size

Change-Id: I3bbd9e7a46a970eb0d9e99d1258b7b27407c0d90
Implements: blueprint qinling-container-resource-limitation
This commit is contained in:
Lingxian Kong 2017-11-30 15:10:37 +13:00
parent 6c013ddb7b
commit 09f5e6dc60
7 changed files with 175 additions and 29 deletions

View File

@ -0,0 +1,38 @@
# Copyright 2017 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import resource
def main(number=1024, **kwargs):
for name, desc in [
('RLIMIT_NOFILE', 'number of open files'),
]:
limit_num = getattr(resource, name)
soft, hard = resource.getrlimit(limit_num)
print('Maximum %-25s (%-15s) : %20s %20s' % (desc, name, soft, hard))
files = []
try:
for i in range(0, number):
files.append(_create_file(i))
finally:
for f in files:
f.close()
def _create_file(index):
f = open('file_%s' % index, 'w')
return f

View File

@ -0,0 +1,43 @@
# Copyright 2017 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from multiprocessing import Process
import resource
import time
def main(number=128, **kwargs):
for name, desc in [
('RLIMIT_NPROC', 'number of processes'),
]:
limit_num = getattr(resource, name)
soft, hard = resource.getrlimit(limit_num)
print('Maximum %-25s (%-15s) : %20s %20s' % (desc, name, soft, hard))
processes = []
for i in range(0, number):
p = Process(
target=_sleep,
args=(i,)
)
p.start()
processes.append(p)
for p in processes:
p.join()
def _sleep(index):
time.sleep(10)

View File

@ -12,9 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import pkg_resources
import tempfile
import zipfile
from oslo_serialization import jsonutils
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
@ -48,19 +50,10 @@ class ExecutionsTest(base.BaseQinlingTest):
super(ExecutionsTest, cls).resource_cleanup()
def setUp(self):
super(ExecutionsTest, self).setUp()
# Wait until runtime is available
self.await_runtime_available(self.runtime_id)
python_file_path = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
os.pardir,
os.pardir,
'functions/python_test.py'
)
def _create_function(self, name='python_test.py'):
python_file_path = pkg_resources.resource_filename(
'qinling_tempest_plugin',
"functions/%s" % name
)
base_name, extention = os.path.splitext(python_file_path)
self.base_name = os.path.basename(base_name)
@ -96,9 +89,13 @@ class ExecutionsTest(base.BaseQinlingTest):
self.addCleanup(self.client.delete_resource, 'functions',
self.function_id, ignore_notfound=True)
self.addCleanup(os.remove, self.python_zip_file)
@decorators.idempotent_id('2a93fab0-2dae-4748-b0d4-f06b735ff451')
def test_crud_execution(self):
self.await_runtime_available(self.runtime_id)
self._create_function()
resp, body = self.client.create_execution(self.function_id,
input={'name': 'Qinling'})
@ -125,6 +122,9 @@ class ExecutionsTest(base.BaseQinlingTest):
@decorators.idempotent_id('8096cc52-64d2-4660-a657-9ac0bdd743ae')
def test_execution_async(self):
self.await_runtime_available(self.runtime_id)
self._create_function()
resp, body = self.client.create_execution(self.function_id, sync=False)
self.assertEqual(201, resp.status)
@ -138,6 +138,9 @@ class ExecutionsTest(base.BaseQinlingTest):
@decorators.idempotent_id('6cb47b1d-a8c6-48f2-a92f-c4f613c33d1c')
def test_execution_log(self):
self.await_runtime_available(self.runtime_id)
self._create_function()
resp, body = self.client.create_execution(self.function_id,
input={'name': 'OpenStack'})
@ -153,3 +156,31 @@ class ExecutionsTest(base.BaseQinlingTest):
self.assertEqual(200, resp.status)
self.assertIn('Hello, OpenStack', body)
def test_python_execution_file_limit(self):
self.await_runtime_available(self.runtime_id)
self._create_function(name='test_python_file_limit.py')
resp, body = self.client.create_execution(self.function_id)
self.assertEqual(201, resp.status)
self.assertEqual('failed', body['status'])
output = jsonutils.loads(body['output'])
self.assertIn(
'Too many open files', output['output']
)
def test_python_execution_process_number(self):
self.await_runtime_available(self.runtime_id)
self._create_function(name='test_python_process_limit.py')
resp, body = self.client.create_execution(self.function_id)
self.assertEqual(201, resp.status)
self.assertEqual('failed', body['status'])
output = jsonutils.loads(body['output'])
self.assertIn(
'too much resource consumption', output['output']
)

View File

@ -1,13 +1,17 @@
FROM phusion/baseimage:0.9.22
MAINTAINER lingxian.kong@gmail.com
MAINTAINER anlin.kong@gmail.com
RUN apt-get update
RUN apt-get -y install python-dev python-setuptools libffi-dev libxslt1-dev libxml2-dev libyaml-dev libssl-dev python-pip
RUN pip install -U pip setuptools
USER root
RUN useradd -Ms /bin/bash qinling
RUN apt-get update && \
apt-get -y install python-dev python-setuptools libffi-dev libxslt1-dev libxml2-dev libyaml-dev libssl-dev python-pip && \
pip install -U pip setuptools
COPY . /app
WORKDIR /app
RUN pip install -r requirements.txt
RUN pip install -r requirements.txt && \
chmod 0750 custom-entrypoint.sh && \
chown -R qinling:qinling /app
ENTRYPOINT ["python", "-u"]
CMD ["server.py"]
CMD ["/bin/bash", "custom-entrypoint.sh"]

View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
# This is expected to run as root for setting the ulimits
set -e
# ensure increased ulimits - for nofile - for the runtime containers
# the limit on the number of files that a single process can have open at a time
ulimit -n 1024
# ensure increased ulimits - for nproc - for the runtime containers
# the limit on the number of processes
ulimit -u 128
# ensure increased ulimits - for file size - for the runtime containers
# the limit on the total file size that a single process can create, 30M
ulimit -f 61440
/sbin/setuser qinling python -u server.py

View File

@ -7,3 +7,4 @@ python-zaqarclient>=1.0.0 # Apache-2.0
python-octaviaclient>=1.0.0 # Apache-2.0
python-mistralclient>=3.1.0 # Apache-2.0
keystoneauth1>=2.21.0 # Apache-2.0
openstacksdk>=0.9.19

View File

@ -88,6 +88,12 @@ def download():
return 'success'
def _print_trace():
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
print(''.join(line for line in lines))
def _invoke_function(execution_id, zip_file, module_name, method, input,
return_dict):
"""Thie function is supposed to be running in a child process."""
@ -102,13 +108,13 @@ def _invoke_function(execution_id, zip_file, module_name, method, input,
return_dict['result'] = func(**input)
return_dict['success'] = True
except Exception as e:
if isinstance(e, OSError) and 'Resource' in str(e):
sys.exit(1)
return_dict['result'] = str(e)
return_dict['success'] = False
# Print stacktrace
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
print(''.join(line for line in lines))
_print_trace()
finally:
print('Finished execution: %s' % execution_id)
@ -119,8 +125,10 @@ def execute():
Several things need to handle in this function:
- Save the function log
- Capture the function exception
- Deal with process execution error
- Capture the function internal exception
- Deal with process execution error (The process may be killed for some
reason, e.g. unlimited memory allocation)
- Deal with os error for process (e.g. Resource temporarily unavailable)
"""
global zip_file
@ -170,12 +178,15 @@ def execute():
p.join()
duration = round(time.time() - start, 3)
output = return_dict.get('result')
# Process was killed unexpectedly.
# Process was killed unexpectedly or finished with error.
if p.exitcode != 0:
output = "Function execution failed because of too much resource " \
"consumption."
success = False
else:
output = return_dict.get('result')
success = return_dict['success']
# Execution log
with open('%s.out' % execution_id) as f:
@ -188,7 +199,7 @@ def execute():
'output': output,
'duration': duration,
'logs': logs,
'success': return_dict['success']
'success': success
}
),
status=200,