Support py3 execution environment

Change-Id: If14d17d74e57eaa1ebe04af9fdc8ef0423c3a989
This commit is contained in:
Kota Tsuyuzaki 2019-03-05 20:54:40 +09:00
parent 998b22ac6c
commit 1a7b0baba6
19 changed files with 223 additions and 52 deletions

View File

@ -0,0 +1,34 @@
# Copyright (c) 2010-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
class BrokenStorlet(object):
def __init__(self, logger):
self.logger = logger
def __call__(self, in_files, out_files, params):
"""
The function called for storlet invocation
:param in_files: a list of StorletInputFile
:param out_files: a list of StorletOutputFile
:param params: a dict of request parameters
"""
# xrange is deprecated in py3
xrange(1) # noqa
# os.scandir is present since py35
# https://docs.python.org/3/whatsnew/3.5.html
os.scandir()

View File

@ -0,0 +1 @@
abcdefghijklmonp

View File

@ -200,6 +200,8 @@ prepare_storlets_install() {
sudo apt-get install -y ant
sudo apt-get install -y python
sudo apt-get install -y python-setuptools
sudo apt-get install -y python3.5
sudo apt-get install -y python3-setuptools
}
_generate_jre_dockerfile() {
@ -209,6 +211,7 @@ MAINTAINER root
RUN apt-get update && \
apt-get install python -y && \
apt-get install python3.5 -y && \
apt-get install git -y && \
apt-get update && \
apt-get install openjdk-8-jre-headless -y && \
@ -288,6 +291,8 @@ install_storlets_code() {
sudo ./install_libs.sh
sudo pip install -r requirements.txt
sudo python setup.py install
sudo pip3 install -r requirements.txt
sudo python3 setup.py install
sudo chown -R ${STORLETS_SWIFT_RUNTIME_USER} storlets.egg-info*
sudo mkdir -p $STORLETS_DOCKER_DEVICE/scripts

View File

@ -54,7 +54,8 @@ Either "Python" or "Java" is available for the value.
The X-Object-Meta-Storlet-Interface-Version header should be provided and set to the value '1.0'.
Although not currently used, the X-Object-Meta-Storlet-Object-Metadata header must be provided and set to 'no'.
See the Storlets Developer's manual for details of the signature of the invoke method.
The content-type of the request should be set to 'application/octet-stream'.
The content-type of the request should be set to 'application/octet-stream'. Only in Python, you may
set 'X-Object-Meta-Storlet-Language-Version' to choose your python interpreter version.
::
@ -76,6 +77,7 @@ For Python written storlets
::
'X-Object-Meta-Storlet-Language': 'Python'
'X-Object-Meta-Storlet-Language-Version': '2.7'
'X-Object-Meta-Storlet-Interface-Version': '1.0'
'X-Object-Meta-Storlet-Dependency': dependencies
'X-Object-Meta-Storlet-Object-Metadata': 'no'

View File

@ -6,5 +6,5 @@ script_dir = /home/docker_device/scripts
storlets_dir = /home/docker_device/storlets/scopes
pipes_dir = /home/docker_device/pipes/scopes
docker_repo = localhost:5001
restart_linux_container_timeout = 3
restart_linux_container_timeout = 10
storlet_timeout = 40

View File

@ -15,6 +15,9 @@
import logging
from logging.handlers import SysLogHandler
DEFAULT_PY2 = 2.7
DEFAULT_PY3 = 3.5
def get_logger(logger_name, log_level, container_id):
"""

View File

@ -26,7 +26,7 @@ from storlets.sbus.command import SBUS_CMD_HALT, SBUS_CMD_PING
from storlets.sbus.file_description import SBUS_FD_SERVICE_OUT
from storlets.agent.common.server import command_handler, CommandSuccess, \
CommandFailure, SBusServer
from storlets.agent.common.utils import get_logger
from storlets.agent.common.utils import get_logger, DEFAULT_PY2, DEFAULT_PY3
class SDaemonError(Exception):
@ -100,10 +100,17 @@ class StorletDaemonFactory(SBusServer):
return pargs, env
def get_python_args(self, daemon_language, storlet_path, storlet_name,
pool_size, uds_path, log_level):
pool_size, uds_path, log_level, language_version):
language_version = language_version or 2
if int(float(language_version)) == 3:
language_version = DEFAULT_PY3
else:
language_version = DEFAULT_PY2
python_interpreter = '/usr/bin/python%s' % language_version
str_daemon_main_file = '/usr/local/libexec/storlets/storlets-daemon'
pargs = [str_daemon_main_file, storlet_name, uds_path, log_level,
str(pool_size), self.container_id]
pargs = [python_interpreter, str_daemon_main_file, storlet_name,
uds_path, log_level, str(pool_size), self.container_id]
python_path = os.path.join('/home/swift/', storlet_name)
if os.environ.get('PYTHONPATH'):
@ -199,7 +206,8 @@ class StorletDaemonFactory(SBusServer):
os.close(write_fd)
def process_start_daemon(self, daemon_language, storlet_path, storlet_name,
pool_size, uds_path, log_level):
pool_size, uds_path, log_level,
language_version=None):
"""
Start storlet daemon process
@ -211,6 +219,8 @@ class StorletDaemonFactory(SBusServer):
pool provides
:param uds_path: Path to pipe daemon is going to listen to
:param log_level: Logger verbosity level
:param language_version: daemon language version (e.g. py2, py3)
only python lang supports this option
:returns: True if it starts a new subprocess
False if there already exists a running process
@ -222,7 +232,7 @@ class StorletDaemonFactory(SBusServer):
elif daemon_language.lower() == 'python':
pargs, env = self.get_python_args(
daemon_language, storlet_path, storlet_name,
pool_size, uds_path, log_level)
pool_size, uds_path, log_level, language_version)
else:
raise SDaemonError(
'Got unsupported daemon language: %s' % daemon_language)
@ -439,7 +449,8 @@ class StorletDaemonFactory(SBusServer):
if self.process_start_daemon(
params['daemon_language'], params['storlet_path'],
storlet_name, params['pool_size'],
params['uds_path'], params['log_level']):
params['uds_path'], params['log_level'],
language_version=params.get("language_version")):
msg = 'OK'
else:
msg = '{0} is already running'.format(storlet_name)

View File

@ -17,6 +17,7 @@
import os
import shutil
from storlets.agent.common.utils import DEFAULT_PY2, DEFAULT_PY3
from storlets.gateway.common.stob import StorletRequest
from storlets.gateway.gateways.base import StorletGatewayBase
from storlets.gateway.gateways.docker.runtime import RunTimePaths, \
@ -125,6 +126,16 @@ class StorletGatewayDocker(StorletGatewayBase):
if '-' not in name or '.' not in name:
raise ValueError('Storlet name is incorrect')
elif params['Language'].lower() == 'python':
# support both py2 and py3
try:
version = int(float(params.get('Language-Version', 2)))
except ValueError:
raise ValueError('Language-Version is invalid')
if version not in [2, DEFAULT_PY2, 3, DEFAULT_PY3]:
# TODO(kota_): more strict version check should be nice.
raise ValueError('Not supported version specified')
if name.endswith('.py'):
cls_name = params['Main']
if not cls_name.startswith(name[:-3] + '.'):

View File

@ -232,7 +232,7 @@ class RunTimeSandbox(object):
self.sandbox_ping_interval = 0.5
self.sandbox_wait_timeout = \
int(conf.get('restart_linux_container_timeout', 3))
int(conf.get('restart_linux_container_timeout', 10))
self.docker_repo = conf.get('docker_repo', '')
self.docker_image_name_prefix = 'tenant'

View File

@ -15,11 +15,13 @@
from six.moves.urllib.parse import unquote
from swift.common.internal_client import InternalClient
from swift.common.swob import HTTPBadRequest, Response, Range
from swift.common.swob import HTTPBadRequest, Response, Range, \
HTTPServiceUnavailable
from swift.common.utils import config_true_value
from storlets.gateway.common.exceptions import FileManagementError
from storlets.gateway.common.file_manager import FileManager
from storlets.gateway.common.exceptions import StorletRuntimeException
class NotStorletRequest(Exception):
@ -418,27 +420,32 @@ class StorletBaseHandler(object):
:param resp: swob.Response instance
:return: processed response
"""
sresp = self._call_gateway(resp)
try:
sresp = self._call_gateway(resp)
new_headers = resp.headers.copy()
new_headers = resp.headers.copy()
if 'Content-Length' in new_headers:
new_headers.pop('Content-Length')
if 'Transfer-Encoding' in new_headers:
new_headers.pop('Transfer-Encoding')
if 'Content-Length' in new_headers:
new_headers.pop('Content-Length')
if 'Transfer-Encoding' in new_headers:
new_headers.pop('Transfer-Encoding')
# Range response(206) should be replaced by 200
# If the range is being processed on the object node
# then we will get 200 as the response will not have a
# range iter.
if 'Content-Range' in resp.headers:
new_headers['Storlet-Input-Range'] = resp.headers['Content-Range']
new_headers.pop('Content-Range')
# Range response(206) should be replaced by 200
# If the range is being processed on the object node
# then we will get 200 as the response will not have a
# range iter.
if 'Content-Range' in resp.headers:
new_headers['Storlet-Input-Range'] = \
resp.headers['Content-Range']
new_headers.pop('Content-Range')
self._set_metadata_in_headers(new_headers, sresp.user_metadata)
self._set_metadata_in_headers(new_headers, sresp.user_metadata)
response = Response(headers=new_headers, app_iter=sresp.data_iter,
reuqest=self.request)
except StorletRuntimeException:
response = HTTPServiceUnavailable()
return Response(headers=new_headers, app_iter=sresp.data_iter,
reuqest=self.request)
return response
def _get_user_metadata(self, headers):
metadata = {}

View File

@ -39,7 +39,7 @@ def put_local_file(url, token, container, local_dir, local_file, headers=None):
def put_storlet_object(url, token, storlet, dependencies, storlet_main_class,
language='Java'):
language='Java', version=None):
"""
Put storlet file to swift
@ -49,6 +49,7 @@ def put_storlet_object(url, token, storlet, dependencies, storlet_main_class,
:param dependencies: a list of dependency files
:param storlet_main_class: name of the storlet main class
:param language: storlet language. default value is Java
:param version: storlet language version. defaulte is 2.7 for python
"""
headers = {'X-Object-Meta-Storlet-Language': language,
'X-Object-Meta-Storlet-Interface-Version': '1.0',
@ -56,6 +57,9 @@ def put_storlet_object(url, token, storlet, dependencies, storlet_main_class,
'X-Object-Meta-Storlet-Main': storlet_main_class}
if dependencies:
headers['X-Object-Meta-Storlet-Dependency'] = dependencies
if version and language.lower() == 'python':
headers['X-Object-Meta-Storlet-Language-Version'] = version
put_local_file(url, token, 'storlet', os.path.dirname(storlet),
os.path.basename(storlet), headers)
@ -76,7 +80,7 @@ def put_storlet_executable_dependencies(url, token, deps):
def deploy_storlet(url, token, storlet, storlet_main_class, dependencies,
language='Java'):
language='Java', version=None):
"""
Deploy storlet file and required dependencies as swift objects
@ -85,12 +89,13 @@ def deploy_storlet(url, token, storlet, storlet_main_class, dependencies,
:param storlet: storlet file to be registerd
:param dependencies: a list of dependency files to be registered
:param language: storlet language. default value is Java
:param version: storlet language version. defaulte is 2.7 for python
"""
# No need to create containers every time
# put_storlet_containers(url, token)
put_storlet_object(url, token, storlet,
','.join(os.path.basename(x) for x in dependencies),
storlet_main_class, language)
storlet_main_class, language, version)
put_storlet_executable_dependencies(url, token, dependencies)

View File

@ -76,7 +76,7 @@ class StorletFunctionalTest(StorletBaseFunctionalTest):
def setUp(self, language, path_to_bundle,
storlet_dir,
storlet_name, storlet_main, storlet_file,
dep_names, headers):
dep_names, headers, version=None):
super(StorletFunctionalTest, self).setUp()
self.storlet_dir = storlet_dir
self.storlet_name = storlet_name
@ -95,7 +95,7 @@ class StorletFunctionalTest(StorletBaseFunctionalTest):
deploy_storlet(self.url, self.token,
storlet, self.storlet_main,
self.deps, language)
self.deps, language, version)
self.create_container(self.container)
if self.storlet_file:

View File

@ -19,7 +19,7 @@ from tests.functional import StorletFunctionalTest, PATH_TO_STORLETS
class StorletPythonFunctionalTest(StorletFunctionalTest):
def setUp(self, storlet_dir, storlet_name, storlet_main,
storlet_file, dep_names=None, headers=None):
storlet_file, dep_names=None, headers=None, version=None):
storlet_dir = os.path.join('python', 'storlet_samples', storlet_dir)
path_to_bundle = os.path.join(PATH_TO_STORLETS, storlet_dir)
super(StorletPythonFunctionalTest, self).setUp('Python',
@ -29,4 +29,5 @@ class StorletPythonFunctionalTest(StorletFunctionalTest):
storlet_main,
storlet_file,
dep_names,
headers)
headers,
version)

View File

@ -0,0 +1,52 @@
# Copyright (c) 2010-2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from swiftclient import client
from swiftclient.exceptions import ClientException
from tests.functional.python import StorletPythonFunctionalTest
import unittest
from storlets.agent.common.utils import DEFAULT_PY3
class TestBrokenStorlet(StorletPythonFunctionalTest):
def setUp(self, version=None):
self.storlet_log = 'broken.log'
self.content = 'abcdefghijklmonp'
self.additional_headers = {}
super(TestBrokenStorlet, self).setUp(
storlet_dir='broken',
storlet_name='broken.py',
storlet_main='broken.BrokenStorlet',
storlet_file='source.txt',
version=version)
def test_get(self):
resp = dict()
req_headers = {'X-Run-Storlet': self.storlet_name}
with self.assertRaises(ClientException) as cm:
client.get_object(
self.url, self.token, self.container, self.storlet_file,
response_dict=resp, headers=req_headers)
e = cm.exception
self.assertEqual(e.http_status, 503)
class TestBrokenStorletRunPy3(TestBrokenStorlet):
def setUp(self):
super(TestBrokenStorletRunPy3, self).setUp(version=DEFAULT_PY3)
if __name__ == '__main__':
unittest.main()

View File

@ -20,10 +20,11 @@ from nose.plugins.attrib import attr
from tests.functional.python import StorletPythonFunctionalTest
import unittest
from eventlet.green import urllib2
from storlets.agent.common.utils import DEFAULT_PY3
class TestSimpleStorlet(StorletPythonFunctionalTest):
def setUp(self):
def setUp(self, version=None):
self.storlet_log = 'simple.log'
self.content = 'abcdefghijklmonp'
self.additional_headers = {}
@ -31,7 +32,8 @@ class TestSimpleStorlet(StorletPythonFunctionalTest):
storlet_dir='simple',
storlet_name='simple.py',
storlet_main='simple.SimpleStorlet',
storlet_file='source.txt')
storlet_file='source.txt',
version=version)
def test_get(self):
resp = dict()
@ -162,5 +164,10 @@ class TestSimpleStorletOnProxy(TestSimpleStorlet):
self.additional_headers = {'X-Storlet-Run-On-Proxy': ''}
class TestSimpleStorletRunPy3(TestSimpleStorlet):
def setUp(self):
super(TestSimpleStorletRunPy3, self).setUp(version=DEFAULT_PY3)
if __name__ == '__main__':
unittest.main()

View File

@ -20,6 +20,7 @@ import unittest
from storlets.sbus import command as sbus_cmd
from storlets.agent.daemon_factory.server import SDaemonError, \
StorletDaemonFactory
from storlets.agent.common.utils import DEFAULT_PY2, DEFAULT_PY3
from tests.unit import FakeLogger
from tests.unit.agent.common import test_server
@ -74,14 +75,21 @@ class TestStorletDaemonFactory(unittest.TestCase):
env)
def test_get_python_args(self):
self._test_get_python_args(DEFAULT_PY2, DEFAULT_PY2)
self._test_get_python_args(2, DEFAULT_PY2)
self._test_get_python_args(DEFAULT_PY3, DEFAULT_PY3)
self._test_get_python_args(3, DEFAULT_PY3)
def _test_get_python_args(self, version, expected):
dummy_env = {'PYTHONPATH': '/default/pythonpath'}
with mock.patch('storlets.agent.daemon_factory.server.os.environ',
dummy_env):
pargs, env = self.dfactory.get_python_args(
'python', 'path/to/storlet', 'test_storlet.TestStorlet',
1, 'path/to/uds', 'DEBUG')
1, 'path/to/uds', 'DEBUG', version)
self.assertEqual(
['/usr/local/libexec/storlets/storlets-daemon',
['/usr/bin/python%s' % expected,
'/usr/local/libexec/storlets/storlets-daemon',
'test_storlet.TestStorlet',
'path/to/uds', 'DEBUG', '1', self.container_id],
pargs)

View File

@ -276,12 +276,24 @@ use = egg:swift#catch_errors
# correct name and headers w/ dependency
obj = 'storlet.py'
params = {'Language': 'python',
'Language-Version': '2.7',
'Interface-Version': '1.0',
'Dependency': 'dep_file',
'Object-Metadata': 'no',
'Main': 'storlet.Storlet'}
StorletGatewayDocker.validate_storlet_registration(params, obj)
# wrong version
obj = 'storlet.py'
params = {'Language': 'python',
'Language-Version': '1.7',
'Interface-Version': '1.0',
'Dependency': 'dep_file',
'Object-Metadata': 'no',
'Main': 'storlet.Storlet'}
with self.assertRaises(ValueError):
StorletGatewayDocker.validate_storlet_registration(params, obj)
# wrong name
obj = 'storlet.pyfoo'
params = {'Language': 'python',

View File

@ -367,13 +367,13 @@ class TestStorletInvocationProtocol(unittest.TestCase):
self.pipe_path = tempfile.mktemp()
self.log_file = tempfile.mktemp()
self.logger = FakeLogger()
storlet_id = 'Storlet-1.0.jar'
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
self.storlet_id = 'Storlet-1.0.jar'
self.options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
storlet_request = DockerStorletRequest(
storlet_id, {}, {}, iter(StringIO()), options=options)
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
self.protocol = StorletInvocationProtocol(
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
@ -422,20 +422,15 @@ class TestStorletInvocationProtocol(unittest.TestCase):
def test_invocation_protocol_remote_fds(self):
# In default, we have 5 fds in remote_fds
storlet_id = 'Storlet-1.0.jar'
options = {'storlet_main': 'org.openstack.storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'java',
'file_manager': FakeFileManager('storlet', 'dep')}
storlet_request = DockerStorletRequest(
storlet_id, {}, {}, iter(StringIO()), options=options)
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
protocol = StorletInvocationProtocol(
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
self.assertEqual(5, len(protocol.remote_fds))
# extra_resources expands the remote_fds
storlet_request = DockerStorletRequest(
storlet_id, {}, {}, iter(StringIO()), options=options)
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
protocol = StorletInvocationProtocol(
storlet_request, self.pipe_path, self.log_file, 1, self.logger,
extra_sources=[storlet_request])
@ -443,7 +438,7 @@ class TestStorletInvocationProtocol(unittest.TestCase):
# 2 more extra_resources expands the remote_fds
storlet_request = DockerStorletRequest(
storlet_id, {}, {}, iter(StringIO()), options=options)
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
protocol = StorletInvocationProtocol(
storlet_request, self.pipe_path, self.log_file, 1, self.logger,
extra_sources=[storlet_request] * 3)
@ -490,5 +485,22 @@ class TestStorletInvocationProtocol(unittest.TestCase):
self._test_writer_with_exception(Exception)
class TestStorletInvocationProtocolPython(TestStorletInvocationProtocol):
def setUp(self):
self.pipe_path = tempfile.mktemp()
self.log_file = tempfile.mktemp()
self.logger = FakeLogger()
self.storlet_id = 'Storlet-1.0.py'
self.options = {'storlet_main': 'storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'python',
'language_version': '2.7',
'file_manager': FakeFileManager('storlet', 'dep')}
storlet_request = DockerStorletRequest(
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)
self.protocol = StorletInvocationProtocol(
storlet_request, self.pipe_path, self.log_file, 1, self.logger)
if __name__ == '__main__':
unittest.main()