Merge "Limit resource consumption of function"
This commit is contained in:
commit
ffaab41c06
|
@ -0,0 +1,38 @@
|
||||||
|
# Copyright 2017 Catalyst IT Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import resource
|
||||||
|
|
||||||
|
|
||||||
|
def main(number=1024, **kwargs):
|
||||||
|
for name, desc in [
|
||||||
|
('RLIMIT_NOFILE', 'number of open files'),
|
||||||
|
]:
|
||||||
|
limit_num = getattr(resource, name)
|
||||||
|
soft, hard = resource.getrlimit(limit_num)
|
||||||
|
print('Maximum %-25s (%-15s) : %20s %20s' % (desc, name, soft, hard))
|
||||||
|
|
||||||
|
files = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
for i in range(0, number):
|
||||||
|
files.append(_create_file(i))
|
||||||
|
finally:
|
||||||
|
for f in files:
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _create_file(index):
|
||||||
|
f = open('file_%s' % index, 'w')
|
||||||
|
return f
|
|
@ -0,0 +1,43 @@
|
||||||
|
# Copyright 2017 Catalyst IT Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from multiprocessing import Process
|
||||||
|
import resource
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
def main(number=128, **kwargs):
|
||||||
|
for name, desc in [
|
||||||
|
('RLIMIT_NPROC', 'number of processes'),
|
||||||
|
]:
|
||||||
|
limit_num = getattr(resource, name)
|
||||||
|
soft, hard = resource.getrlimit(limit_num)
|
||||||
|
print('Maximum %-25s (%-15s) : %20s %20s' % (desc, name, soft, hard))
|
||||||
|
|
||||||
|
processes = []
|
||||||
|
|
||||||
|
for i in range(0, number):
|
||||||
|
p = Process(
|
||||||
|
target=_sleep,
|
||||||
|
args=(i,)
|
||||||
|
)
|
||||||
|
p.start()
|
||||||
|
processes.append(p)
|
||||||
|
|
||||||
|
for p in processes:
|
||||||
|
p.join()
|
||||||
|
|
||||||
|
|
||||||
|
def _sleep(index):
|
||||||
|
time.sleep(10)
|
|
@ -12,9 +12,11 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
import os
|
import os
|
||||||
|
import pkg_resources
|
||||||
import tempfile
|
import tempfile
|
||||||
import zipfile
|
import zipfile
|
||||||
|
|
||||||
|
from oslo_serialization import jsonutils
|
||||||
from tempest.lib.common.utils import data_utils
|
from tempest.lib.common.utils import data_utils
|
||||||
from tempest.lib import decorators
|
from tempest.lib import decorators
|
||||||
|
|
||||||
|
@ -48,19 +50,10 @@ class ExecutionsTest(base.BaseQinlingTest):
|
||||||
|
|
||||||
super(ExecutionsTest, cls).resource_cleanup()
|
super(ExecutionsTest, cls).resource_cleanup()
|
||||||
|
|
||||||
def setUp(self):
|
def _create_function(self, name='python_test.py'):
|
||||||
super(ExecutionsTest, self).setUp()
|
python_file_path = pkg_resources.resource_filename(
|
||||||
|
'qinling_tempest_plugin',
|
||||||
# Wait until runtime is available
|
"functions/%s" % name
|
||||||
self.await_runtime_available(self.runtime_id)
|
|
||||||
|
|
||||||
python_file_path = os.path.abspath(
|
|
||||||
os.path.join(
|
|
||||||
os.path.dirname(__file__),
|
|
||||||
os.pardir,
|
|
||||||
os.pardir,
|
|
||||||
'functions/python_test.py'
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
base_name, extention = os.path.splitext(python_file_path)
|
base_name, extention = os.path.splitext(python_file_path)
|
||||||
self.base_name = os.path.basename(base_name)
|
self.base_name = os.path.basename(base_name)
|
||||||
|
@ -96,9 +89,13 @@ class ExecutionsTest(base.BaseQinlingTest):
|
||||||
|
|
||||||
self.addCleanup(self.client.delete_resource, 'functions',
|
self.addCleanup(self.client.delete_resource, 'functions',
|
||||||
self.function_id, ignore_notfound=True)
|
self.function_id, ignore_notfound=True)
|
||||||
|
self.addCleanup(os.remove, self.python_zip_file)
|
||||||
|
|
||||||
@decorators.idempotent_id('2a93fab0-2dae-4748-b0d4-f06b735ff451')
|
@decorators.idempotent_id('2a93fab0-2dae-4748-b0d4-f06b735ff451')
|
||||||
def test_crud_execution(self):
|
def test_crud_execution(self):
|
||||||
|
self.await_runtime_available(self.runtime_id)
|
||||||
|
self._create_function()
|
||||||
|
|
||||||
resp, body = self.client.create_execution(self.function_id,
|
resp, body = self.client.create_execution(self.function_id,
|
||||||
input={'name': 'Qinling'})
|
input={'name': 'Qinling'})
|
||||||
|
|
||||||
|
@ -125,6 +122,9 @@ class ExecutionsTest(base.BaseQinlingTest):
|
||||||
|
|
||||||
@decorators.idempotent_id('8096cc52-64d2-4660-a657-9ac0bdd743ae')
|
@decorators.idempotent_id('8096cc52-64d2-4660-a657-9ac0bdd743ae')
|
||||||
def test_execution_async(self):
|
def test_execution_async(self):
|
||||||
|
self.await_runtime_available(self.runtime_id)
|
||||||
|
self._create_function()
|
||||||
|
|
||||||
resp, body = self.client.create_execution(self.function_id, sync=False)
|
resp, body = self.client.create_execution(self.function_id, sync=False)
|
||||||
|
|
||||||
self.assertEqual(201, resp.status)
|
self.assertEqual(201, resp.status)
|
||||||
|
@ -138,6 +138,9 @@ class ExecutionsTest(base.BaseQinlingTest):
|
||||||
|
|
||||||
@decorators.idempotent_id('6cb47b1d-a8c6-48f2-a92f-c4f613c33d1c')
|
@decorators.idempotent_id('6cb47b1d-a8c6-48f2-a92f-c4f613c33d1c')
|
||||||
def test_execution_log(self):
|
def test_execution_log(self):
|
||||||
|
self.await_runtime_available(self.runtime_id)
|
||||||
|
self._create_function()
|
||||||
|
|
||||||
resp, body = self.client.create_execution(self.function_id,
|
resp, body = self.client.create_execution(self.function_id,
|
||||||
input={'name': 'OpenStack'})
|
input={'name': 'OpenStack'})
|
||||||
|
|
||||||
|
@ -153,3 +156,31 @@ class ExecutionsTest(base.BaseQinlingTest):
|
||||||
|
|
||||||
self.assertEqual(200, resp.status)
|
self.assertEqual(200, resp.status)
|
||||||
self.assertIn('Hello, OpenStack', body)
|
self.assertIn('Hello, OpenStack', body)
|
||||||
|
|
||||||
|
def test_python_execution_file_limit(self):
|
||||||
|
self.await_runtime_available(self.runtime_id)
|
||||||
|
self._create_function(name='test_python_file_limit.py')
|
||||||
|
|
||||||
|
resp, body = self.client.create_execution(self.function_id)
|
||||||
|
|
||||||
|
self.assertEqual(201, resp.status)
|
||||||
|
self.assertEqual('failed', body['status'])
|
||||||
|
|
||||||
|
output = jsonutils.loads(body['output'])
|
||||||
|
self.assertIn(
|
||||||
|
'Too many open files', output['output']
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_python_execution_process_number(self):
|
||||||
|
self.await_runtime_available(self.runtime_id)
|
||||||
|
self._create_function(name='test_python_process_limit.py')
|
||||||
|
|
||||||
|
resp, body = self.client.create_execution(self.function_id)
|
||||||
|
|
||||||
|
self.assertEqual(201, resp.status)
|
||||||
|
self.assertEqual('failed', body['status'])
|
||||||
|
|
||||||
|
output = jsonutils.loads(body['output'])
|
||||||
|
self.assertIn(
|
||||||
|
'too much resource consumption', output['output']
|
||||||
|
)
|
||||||
|
|
|
@ -1,13 +1,17 @@
|
||||||
FROM phusion/baseimage:0.9.22
|
FROM phusion/baseimage:0.9.22
|
||||||
MAINTAINER lingxian.kong@gmail.com
|
MAINTAINER anlin.kong@gmail.com
|
||||||
|
|
||||||
RUN apt-get update
|
USER root
|
||||||
RUN apt-get -y install python-dev python-setuptools libffi-dev libxslt1-dev libxml2-dev libyaml-dev libssl-dev python-pip
|
RUN useradd -Ms /bin/bash qinling
|
||||||
RUN pip install -U pip setuptools
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get -y install python-dev python-setuptools libffi-dev libxslt1-dev libxml2-dev libyaml-dev libssl-dev python-pip && \
|
||||||
|
pip install -U pip setuptools
|
||||||
|
|
||||||
COPY . /app
|
COPY . /app
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
RUN pip install -r requirements.txt
|
RUN pip install -r requirements.txt && \
|
||||||
|
chmod 0750 custom-entrypoint.sh && \
|
||||||
|
chown -R qinling:qinling /app
|
||||||
|
|
||||||
ENTRYPOINT ["python", "-u"]
|
CMD ["/bin/bash", "custom-entrypoint.sh"]
|
||||||
CMD ["server.py"]
|
|
||||||
|
|
|
@ -0,0 +1,18 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# This is expected to run as root for setting the ulimits
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# ensure increased ulimits - for nofile - for the runtime containers
|
||||||
|
# the limit on the number of files that a single process can have open at a time
|
||||||
|
ulimit -n 1024
|
||||||
|
|
||||||
|
# ensure increased ulimits - for nproc - for the runtime containers
|
||||||
|
# the limit on the number of processes
|
||||||
|
ulimit -u 128
|
||||||
|
|
||||||
|
# ensure increased ulimits - for file size - for the runtime containers
|
||||||
|
# the limit on the total file size that a single process can create, 30M
|
||||||
|
ulimit -f 61440
|
||||||
|
|
||||||
|
/sbin/setuser qinling python -u server.py
|
|
@ -7,3 +7,4 @@ python-zaqarclient>=1.0.0 # Apache-2.0
|
||||||
python-octaviaclient>=1.0.0 # Apache-2.0
|
python-octaviaclient>=1.0.0 # Apache-2.0
|
||||||
python-mistralclient>=3.1.0 # Apache-2.0
|
python-mistralclient>=3.1.0 # Apache-2.0
|
||||||
keystoneauth1>=2.21.0 # Apache-2.0
|
keystoneauth1>=2.21.0 # Apache-2.0
|
||||||
|
openstacksdk>=0.9.19
|
||||||
|
|
|
@ -88,6 +88,12 @@ def download():
|
||||||
return 'success'
|
return 'success'
|
||||||
|
|
||||||
|
|
||||||
|
def _print_trace():
|
||||||
|
exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||||
|
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
|
||||||
|
print(''.join(line for line in lines))
|
||||||
|
|
||||||
|
|
||||||
def _invoke_function(execution_id, zip_file, module_name, method, input,
|
def _invoke_function(execution_id, zip_file, module_name, method, input,
|
||||||
return_dict):
|
return_dict):
|
||||||
"""Thie function is supposed to be running in a child process."""
|
"""Thie function is supposed to be running in a child process."""
|
||||||
|
@ -102,13 +108,13 @@ def _invoke_function(execution_id, zip_file, module_name, method, input,
|
||||||
return_dict['result'] = func(**input)
|
return_dict['result'] = func(**input)
|
||||||
return_dict['success'] = True
|
return_dict['success'] = True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
if isinstance(e, OSError) and 'Resource' in str(e):
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
return_dict['result'] = str(e)
|
return_dict['result'] = str(e)
|
||||||
return_dict['success'] = False
|
return_dict['success'] = False
|
||||||
|
|
||||||
# Print stacktrace
|
_print_trace()
|
||||||
exc_type, exc_value, exc_traceback = sys.exc_info()
|
|
||||||
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
|
|
||||||
print(''.join(line for line in lines))
|
|
||||||
finally:
|
finally:
|
||||||
print('Finished execution: %s' % execution_id)
|
print('Finished execution: %s' % execution_id)
|
||||||
|
|
||||||
|
@ -119,8 +125,10 @@ def execute():
|
||||||
|
|
||||||
Several things need to handle in this function:
|
Several things need to handle in this function:
|
||||||
- Save the function log
|
- Save the function log
|
||||||
- Capture the function exception
|
- Capture the function internal exception
|
||||||
- Deal with process execution error
|
- Deal with process execution error (The process may be killed for some
|
||||||
|
reason, e.g. unlimited memory allocation)
|
||||||
|
- Deal with os error for process (e.g. Resource temporarily unavailable)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
global zip_file
|
global zip_file
|
||||||
|
@ -170,12 +178,15 @@ def execute():
|
||||||
p.join()
|
p.join()
|
||||||
|
|
||||||
duration = round(time.time() - start, 3)
|
duration = round(time.time() - start, 3)
|
||||||
output = return_dict.get('result')
|
|
||||||
|
|
||||||
# Process was killed unexpectedly.
|
# Process was killed unexpectedly or finished with error.
|
||||||
if p.exitcode != 0:
|
if p.exitcode != 0:
|
||||||
output = "Function execution failed because of too much resource " \
|
output = "Function execution failed because of too much resource " \
|
||||||
"consumption."
|
"consumption."
|
||||||
|
success = False
|
||||||
|
else:
|
||||||
|
output = return_dict.get('result')
|
||||||
|
success = return_dict['success']
|
||||||
|
|
||||||
# Execution log
|
# Execution log
|
||||||
with open('%s.out' % execution_id) as f:
|
with open('%s.out' % execution_id) as f:
|
||||||
|
@ -188,7 +199,7 @@ def execute():
|
||||||
'output': output,
|
'output': output,
|
||||||
'duration': duration,
|
'duration': duration,
|
||||||
'logs': logs,
|
'logs': logs,
|
||||||
'success': return_dict['success']
|
'success': success
|
||||||
}
|
}
|
||||||
),
|
),
|
||||||
status=200,
|
status=200,
|
||||||
|
|
Loading…
Reference in New Issue