Invoke Python function in a directory rather than a zip package
There is a limitation for loading modules directly from a zip file: "ZIP import of dynamic modules (.pyd, .so) is disallowed." which makes most of python libs including C extentions not work. So we have to extract the zip before executing the function. Change-Id: I4d4eed380a4140c905603891e1dfa942834bd25d
This commit is contained in:
parent
3e27c859e3
commit
bbfe83eee3
|
@ -22,8 +22,8 @@ QINLING_CONF_FILE=${QINLING_CONF_DIR}/qinling.conf
|
||||||
QINLING_POLICY_FILE=${QINLING_CONF_DIR}/policy.json
|
QINLING_POLICY_FILE=${QINLING_CONF_DIR}/policy.json
|
||||||
QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling}
|
QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling}
|
||||||
QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/function/packages}
|
QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/function/packages}
|
||||||
QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime:0.0.3}
|
QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime:0.0.4}
|
||||||
QINLING_NODEJS_RUNTIME_IMAGE=${QINLING_NODEJS_RUNTIME_IMAGE:-openstackqinling/nodejs-runtime:0.0.1}
|
QINLING_NODEJS_RUNTIME_IMAGE=${QINLING_NODEJS_RUNTIME_IMAGE:-openstackqinling/nodejs-runtime:0.0.1}
|
||||||
QINLING_SIDECAR_IMAGE=${QINLING_SIDECAR_IMAGE:-openstackqinling/sidecar:0.0.1}
|
QINLING_SIDECAR_IMAGE=${QINLING_SIDECAR_IMAGE:-openstackqinling/sidecar:0.0.2}
|
||||||
|
|
||||||
QINLING_K8S_APISERVER_TLS=${QINLING_K8S_APISERVER_TLS:-True}
|
QINLING_K8S_APISERVER_TLS=${QINLING_K8S_APISERVER_TLS:-True}
|
||||||
|
|
|
@ -114,8 +114,8 @@ engine_opts = [
|
||||||
),
|
),
|
||||||
cfg.StrOpt(
|
cfg.StrOpt(
|
||||||
'sidecar_image',
|
'sidecar_image',
|
||||||
default='openstackqinling/sidecar:0.0.1',
|
default='openstackqinling/sidecar:0.0.2',
|
||||||
help='The sidecar image being used in the worker.'
|
help='The sidecar image being used together with the worker.'
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
@ -18,8 +18,7 @@ from oslo_config import cfg
|
||||||
service_option = cfg.BoolOpt(
|
service_option = cfg.BoolOpt(
|
||||||
'qinling',
|
'qinling',
|
||||||
default=True,
|
default=True,
|
||||||
help="Whether or not Qinling is expected to be"
|
help="Whether or not Qinling is expected to be available"
|
||||||
"available"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@ -41,7 +40,7 @@ QinlingGroup = [
|
||||||
'publicURL', 'adminURL', 'internalURL'],
|
'publicURL', 'adminURL', 'internalURL'],
|
||||||
help="The endpoint type to use for the qinling service."),
|
help="The endpoint type to use for the qinling service."),
|
||||||
cfg.StrOpt("python_runtime_image",
|
cfg.StrOpt("python_runtime_image",
|
||||||
default="openstackqinling/python-runtime:0.0.3",
|
default="openstackqinling/python-runtime:0.0.4",
|
||||||
help="The Python runtime being used in the tests."),
|
help="The Python runtime being used in the tests."),
|
||||||
cfg.StrOpt("nodejs_runtime_image",
|
cfg.StrOpt("nodejs_runtime_image",
|
||||||
default="openstackqinling/nodejs-runtime:0.0.1",
|
default="openstackqinling/nodejs-runtime:0.0.1",
|
||||||
|
|
|
@ -72,8 +72,8 @@ def _get_responce(output, duration, logs, success, code):
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _invoke_function(execution_id, zip_file, module_name, method, arg, input,
|
def _invoke_function(execution_id, zip_file_dir, module_name, method, arg,
|
||||||
return_dict, rlimit):
|
input, return_dict, rlimit):
|
||||||
"""Thie function is supposed to be running in a child process.
|
"""Thie function is supposed to be running in a child process.
|
||||||
|
|
||||||
HOSTNAME will be used to create cgroup directory related to worker.
|
HOSTNAME will be used to create cgroup directory related to worker.
|
||||||
|
@ -102,7 +102,7 @@ def _invoke_function(execution_id, zip_file, module_name, method, arg, input,
|
||||||
return_dict['success'] = False
|
return_dict['success'] = False
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
sys.path.insert(0, zip_file)
|
sys.path.insert(0, zip_file_dir)
|
||||||
sys.stdout = open("%s.out" % execution_id, "w", 0)
|
sys.stdout = open("%s.out" % execution_id, "w", 0)
|
||||||
|
|
||||||
print('Start execution: %s' % execution_id)
|
print('Start execution: %s' % execution_id)
|
||||||
|
@ -146,7 +146,7 @@ def execute():
|
||||||
auth_url = params.get('auth_url')
|
auth_url = params.get('auth_url')
|
||||||
username = params.get('username')
|
username = params.get('username')
|
||||||
password = params.get('password')
|
password = params.get('password')
|
||||||
zip_file = '/var/qinling/packages/%s.zip' % function_id
|
zip_file_dir = '/var/qinling/packages/%s' % function_id
|
||||||
rlimit = {
|
rlimit = {
|
||||||
'cpu': params['cpu'],
|
'cpu': params['cpu'],
|
||||||
'memory_size': params['memory_size']
|
'memory_size': params['memory_size']
|
||||||
|
@ -209,7 +209,7 @@ def execute():
|
||||||
# Run the function in a separate process to avoid messing up the log
|
# Run the function in a separate process to avoid messing up the log
|
||||||
p = Process(
|
p = Process(
|
||||||
target=_invoke_function,
|
target=_invoke_function,
|
||||||
args=(execution_id, zip_file, function_module, function_method,
|
args=(execution_id, zip_file_dir, function_module, function_method,
|
||||||
input.pop('__function_input', None), input, return_dict, rlimit)
|
input.pop('__function_input', None), input, return_dict, rlimit)
|
||||||
)
|
)
|
||||||
p.start()
|
p.start()
|
||||||
|
|
|
@ -98,7 +98,7 @@ def download():
|
||||||
params['download_url'],
|
params['download_url'],
|
||||||
zip_file,
|
zip_file,
|
||||||
token=params.get('token'),
|
token=params.get('token'),
|
||||||
unzip=params.get('unzip')
|
unzip=params.get('unzip', True)
|
||||||
)
|
)
|
||||||
|
|
||||||
return resp if resp else 'downloaded'
|
return resp if resp else 'downloaded'
|
||||||
|
|
Loading…
Reference in New Issue