Invoke Python function in a directory rather than a zip package

There is a limitation for loading modules directly from a zip file:
"ZIP import of dynamic modules (.pyd, .so) is disallowed." which
makes most of python libs including C extentions not work. So we have
to extract the zip before executing the function.

Change-Id: I4d4eed380a4140c905603891e1dfa942834bd25d
This commit is contained in:
Lingxian Kong 2018-07-03 20:44:12 +12:00
parent 3e27c859e3
commit bbfe83eee3
5 changed files with 12 additions and 13 deletions

View File

@ -22,8 +22,8 @@ QINLING_CONF_FILE=${QINLING_CONF_DIR}/qinling.conf
QINLING_POLICY_FILE=${QINLING_CONF_DIR}/policy.json
QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling}
QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/function/packages}
QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime:0.0.3}
QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime:0.0.4}
QINLING_NODEJS_RUNTIME_IMAGE=${QINLING_NODEJS_RUNTIME_IMAGE:-openstackqinling/nodejs-runtime:0.0.1}
QINLING_SIDECAR_IMAGE=${QINLING_SIDECAR_IMAGE:-openstackqinling/sidecar:0.0.1}
QINLING_SIDECAR_IMAGE=${QINLING_SIDECAR_IMAGE:-openstackqinling/sidecar:0.0.2}
QINLING_K8S_APISERVER_TLS=${QINLING_K8S_APISERVER_TLS:-True}

View File

@ -114,8 +114,8 @@ engine_opts = [
),
cfg.StrOpt(
'sidecar_image',
default='openstackqinling/sidecar:0.0.1',
help='The sidecar image being used in the worker.'
default='openstackqinling/sidecar:0.0.2',
help='The sidecar image being used together with the worker.'
),
]

View File

@ -18,8 +18,7 @@ from oslo_config import cfg
service_option = cfg.BoolOpt(
'qinling',
default=True,
help="Whether or not Qinling is expected to be"
"available"
help="Whether or not Qinling is expected to be available"
)
@ -41,7 +40,7 @@ QinlingGroup = [
'publicURL', 'adminURL', 'internalURL'],
help="The endpoint type to use for the qinling service."),
cfg.StrOpt("python_runtime_image",
default="openstackqinling/python-runtime:0.0.3",
default="openstackqinling/python-runtime:0.0.4",
help="The Python runtime being used in the tests."),
cfg.StrOpt("nodejs_runtime_image",
default="openstackqinling/nodejs-runtime:0.0.1",

View File

@ -72,8 +72,8 @@ def _get_responce(output, duration, logs, success, code):
)
def _invoke_function(execution_id, zip_file, module_name, method, arg, input,
return_dict, rlimit):
def _invoke_function(execution_id, zip_file_dir, module_name, method, arg,
input, return_dict, rlimit):
"""Thie function is supposed to be running in a child process.
HOSTNAME will be used to create cgroup directory related to worker.
@ -102,7 +102,7 @@ def _invoke_function(execution_id, zip_file, module_name, method, arg, input,
return_dict['success'] = False
sys.exit(0)
sys.path.insert(0, zip_file)
sys.path.insert(0, zip_file_dir)
sys.stdout = open("%s.out" % execution_id, "w", 0)
print('Start execution: %s' % execution_id)
@ -146,7 +146,7 @@ def execute():
auth_url = params.get('auth_url')
username = params.get('username')
password = params.get('password')
zip_file = '/var/qinling/packages/%s.zip' % function_id
zip_file_dir = '/var/qinling/packages/%s' % function_id
rlimit = {
'cpu': params['cpu'],
'memory_size': params['memory_size']
@ -209,7 +209,7 @@ def execute():
# Run the function in a separate process to avoid messing up the log
p = Process(
target=_invoke_function,
args=(execution_id, zip_file, function_module, function_method,
args=(execution_id, zip_file_dir, function_module, function_method,
input.pop('__function_input', None), input, return_dict, rlimit)
)
p.start()

View File

@ -98,7 +98,7 @@ def download():
params['download_url'],
zip_file,
token=params.get('token'),
unzip=params.get('unzip')
unzip=params.get('unzip', True)
)
return resp if resp else 'downloaded'