openstack
/
storlets
Code Issues Proposed changes

Drop Python 2 support in storlet agent 

This change removes support running storlet agent using Python 2, which
means we no longer support running applications by Python 2.

This patch also cleans up some codes we have for py2-3 compatibility,
which is no longer required after we drop support for Python 2.

Change-Id: Ib03e8d442ad3362fd9f2bbc0f1c09f597dad684c
This commit is contained in:
Takashi Kajinami 2020-02-24 00:19:47 +09:00
parent bce81a9597
commit 132d3509f2
13 changed files with 32 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
devstack/plugin.sh
View File

@ -181,21 +181,11 @@ function prepare_storlets_install {
die $LINENO "Unsupported distro"
fi
if [ "${USE_PYTHON3}" == "False" ]; then
# TODO(takashi): Remove this when we remove py2 support
install_package python2.7 python2.7-dev
else
install_python3
fi
install_python3
}
function _generate_jre_dockerfile {
# TODO(tkajinam): Remove py2 packages when we remove its support
local PYTHON_PACKAGES='python2.7 python2.7-dev python3.8 python3.8-dev'
if python3_enabled; then
PYTHON_PACKAGES="python2.7 python2.7-dev python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev"
fi
PYTHON_PACKAGES="python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev"
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/${STORLETS_DOCKER_BASE_IMG_NAME}_jre${STORLETS_JDK_VERSION}/Dockerfile
FROM $STORLETS_DOCKER_BASE_IMG

2
doc/source/api/overview_api.rst
View File

@ -77,7 +77,7 @@ For Python written storlets
::
'X-Object-Meta-Storlet-Language': 'Python'
'X-Object-Meta-Storlet-Language-Version': '2.7'
'X-Object-Meta-Storlet-Language-Version': '3.6'
'X-Object-Meta-Storlet-Interface-Version': '1.0'
'X-Object-Meta-Storlet-Dependency': dependencies
'X-Object-Meta-Storlet-Object-Metadata': 'no'

3
storlets/agent/common/utils.py
View File

@ -15,8 +15,7 @@
import logging
from logging.handlers import SysLogHandler
# TODO(takashi): Make the following parameters configurable
DEFAULT_PY2 = 2.7
# TODO(takashi): Make the following parameter configurable
DEFAULT_PY3 = 3.8

11
storlets/agent/daemon_factory/server.py
View File

@ -26,7 +26,7 @@ from storlets.sbus.client.exceptions import SBusClientException, \
SBusClientSendError
from storlets.agent.common.server import command_handler, EXIT_FAILURE, \
CommandSuccess, CommandFailure, SBusServer
from storlets.agent.common.utils import get_logger, DEFAULT_PY2, DEFAULT_PY3
from storlets.agent.common.utils import get_logger, DEFAULT_PY3
class SDaemonError(Exception):
@ -103,12 +103,9 @@ class StorletDaemonFactory(SBusServer):
def get_python_args(self, daemon_language, storlet_path, storlet_name,
pool_size, uds_path, log_level,
daemon_language_version):
daemon_language_version = daemon_language_version or 3
# TODO(takashi): Drop Py2 support
if int(float(daemon_language_version)) == 2:
daemon_language_version = DEFAULT_PY2
else:
daemon_language_version = DEFAULT_PY3
# TODO(takashi): We currently override all 3.x by DEFAULT_PY3, but
# we might need to support multiple minor versions.
daemon_language_version = DEFAULT_PY3
python_interpreter = '/usr/bin/python%s' % daemon_language_version
str_daemon_main_file = '/usr/local/libexec/storlets/storlets-daemon'

5
storlets/gateway/gateways/docker/gateway.py
View File

@ -17,7 +17,7 @@
import os
import shutil
from storlets.agent.common.utils import DEFAULT_PY2, DEFAULT_PY3
from storlets.agent.common.utils import DEFAULT_PY3
from storlets.gateway.common.stob import StorletRequest
from storlets.gateway.gateways.base import StorletGatewayBase
from storlets.gateway.gateways.docker.runtime import RunTimePaths, \
@ -133,8 +133,7 @@ class StorletGatewayDocker(StorletGatewayBase):
except ValueError:
raise ValueError('Language-Version is invalid')
# TODO(takashi): Drop Py2 support
if version not in [2, DEFAULT_PY2, 3, DEFAULT_PY3]:
if version not in [3, DEFAULT_PY3]:
# TODO(kota_): more strict version check should be nice.
raise ValueError('Not supported version specified')

5
storlets/sbus/client/client.py
View File

@ -93,10 +93,7 @@ class SBusClient(object):
finally:
os.close(read_fd)
if not isinstance(reply, str):
reply = reply.decode('utf-8')
return self._parse_response(reply)
return self._parse_response(reply.decode('utf-8'))
def execute(self, invocation_params, invocation_fds):
return self._request(sbus_cmd.SBUS_CMD_EXECUTE,

10
storlets/sbus/sbus.py
View File

@ -107,15 +107,9 @@ class SBus(object):
# Extract Python strings
n_metadata = pn_metadata.value
# NOTE: because the storlets container may not have
# six module inside. Just check the type, not the python
# version. Anyway, the value should be bytes but python2
# can assume the bytes as str.
str_metadata = pp_metadata.value.decode("utf-8") \
if not isinstance(pp_metadata.value, str) else pp_metadata.value
str_metadata = pp_metadata.value.decode("utf-8")
n_params = pn_params.value
str_params = pp_params.value.decode("utf-8") \
if not isinstance(pp_params.value, str) else pp_params.value
str_params = pp_params.value.decode("utf-8")
# Trim the junk out
if 0 < n_metadata:

4
storlets/tools/utils.py
View File

@ -49,7 +49,7 @@ def put_storlet_object(url, token, storlet, dependencies, storlet_main_class,
:param dependencies: a list of dependency files
:param storlet_main_class: name of the storlet main class
:param language: storlet language. default value is Java
:param version: storlet language version. defaulte is 2.7 for python
:param version: storlet language version. default is 3.8 for python
"""
headers = {'X-Object-Meta-Storlet-Language': language,
'X-Object-Meta-Storlet-Interface-Version': '1.0',
@ -89,7 +89,7 @@ def deploy_storlet(url, token, storlet, storlet_main_class, dependencies,
:param storlet: storlet file to be registerd
:param dependencies: a list of dependency files to be registered
:param language: storlet language. default value is Java
:param version: storlet language version. defaulte is 2.7 for python
:param version: storlet language version. default is 3.8 for python
"""
# No need to create containers every time
# put_storlet_containers(url, token)

6
tests/functional/python/test_broken_storlet.py
View File

@ -17,7 +17,6 @@ from swiftclient import client
from swiftclient.exceptions import ClientException
from tests.functional.python import StorletPythonFunctionalTest
import unittest
from storlets.agent.common.utils import DEFAULT_PY2
class TestBrokenStorlet(StorletPythonFunctionalTest):
@ -43,10 +42,5 @@ class TestBrokenStorlet(StorletPythonFunctionalTest):
self.assertEqual(e.http_status, 503)
class TestBrokenStorletRunPy2(TestBrokenStorlet):
def setUp(self):
super(TestBrokenStorletRunPy2, self).setUp(version=DEFAULT_PY2)
if __name__ == '__main__':
unittest.main()

6
tests/functional/python/test_simple_storlet.py
View File

@ -20,7 +20,6 @@ from nose.plugins.attrib import attr
from tests.functional.python import StorletPythonFunctionalTest
import unittest
from six.moves.urllib.request import Request, urlopen
from storlets.agent.common.utils import DEFAULT_PY2
class TestSimpleStorlet(StorletPythonFunctionalTest):
@ -164,10 +163,5 @@ class TestSimpleStorletOnProxy(TestSimpleStorlet):
self.additional_headers = {'X-Storlet-Run-On-Proxy': ''}
class TestSimpleStorletRunPy2(TestSimpleStorlet):
def setUp(self):
super(TestSimpleStorletRunPy2, self).setUp(version=DEFAULT_PY2)
if __name__ == '__main__':
unittest.main()

4
tests/unit/agent/daemon_factory/test_server.py
View File

@ -23,7 +23,7 @@ from storlets.sbus.client.exceptions import SBusClientSendError
from storlets.agent.daemon_factory.server import SDaemonError, \
StorletDaemonFactory
from storlets.agent.common.utils import DEFAULT_PY2, DEFAULT_PY3
from storlets.agent.common.utils import DEFAULT_PY3
from tests.unit import FakeLogger
from tests.unit.agent.common import test_server
@ -89,8 +89,6 @@ class TestStorletDaemonFactory(unittest.TestCase):
def test_get_python_args(self):
self._test_get_python_args(None, DEFAULT_PY3)
self._test_get_python_args(DEFAULT_PY2, DEFAULT_PY2)
self._test_get_python_args(2, DEFAULT_PY2)
self._test_get_python_args(DEFAULT_PY3, DEFAULT_PY3)
self._test_get_python_args(3, DEFAULT_PY3)

17
tests/unit/gateway/gateways/docker/test_gateway.py
View File

@ -120,7 +120,7 @@ class TestDockerStorletRequest(unittest.TestCase):
# with language version
options = {'storlet_main': 'storlet.Storlet',
'storlet_language': 'python',
'storlet_language_version': '2.7',
'storlet_language_version': '3.6',
'file_manager': FakeFileManager('storlet', 'dep')}
dsreq = DockerStorletRequest(storlet_id, params, metadata,
iter(StringIO()), options=options)
@ -130,7 +130,7 @@ class TestDockerStorletRequest(unittest.TestCase):
self.assertEqual('storlet.Storlet', dsreq.storlet_main)
self.assertEqual([], dsreq.dependencies)
self.assertEqual('python', dsreq.storlet_language)
self.assertEqual('2.7', dsreq.storlet_language_version)
self.assertEqual('3.6', dsreq.storlet_language_version)
def test_init_with_range(self):
storlet_id = 'Storlet-1.0.jar'
@ -324,7 +324,7 @@ use = egg:swift#catch_errors
# correct name and headers w/ dependency
obj = 'storlet.py'
params = {'Language': 'python',
'Language-Version': '2.7',
'Language-Version': '3.6',
'Interface-Version': '1.0',
'Dependency': 'dep_file',
'Object-Metadata': 'no',
@ -342,6 +342,17 @@ use = egg:swift#catch_errors
with self.assertRaises(ValueError):
StorletGatewayDocker.validate_storlet_registration(params, obj)
# py2 is no more supported
obj = 'storlet.py'
params = {'Language': 'python',
'Language-Version': '2.7',
'Interface-Version': '1.0',
'Dependency': 'dep_file',
'Object-Metadata': 'no',
'Main': 'storlet.Storlet'}
with self.assertRaises(ValueError):
StorletGatewayDocker.validate_storlet_registration(params, obj)
# wrong name
obj = 'storlet.pyfoo'
params = {'Language': 'python',

4
tests/unit/gateway/gateways/docker/test_runtime.py
View File

@ -42,7 +42,7 @@ from tests.unit.gateway.gateways import FakeFileManager
def _mock_os_pipe(bufs):
class FakeFd(object):
def __init__(self, rbuf=''):
self.rbuf = rbuf
self.rbuf = rbuf.encode('utf-8')
self.closed = False
def read(self, size):
@ -640,7 +640,7 @@ class TestStorletInvocationProtocolPython(TestStorletInvocationProtocol):
self.options = {'storlet_main': 'storlet.Storlet',
'storlet_dependency': 'dep1,dep2',
'storlet_language': 'python',
'language_version': '2.7',
'language_version': '3.6',
'file_manager': FakeFileManager('storlet', 'dep')}
storlet_request = DockerStorletRequest(
self.storlet_id, {}, {}, iter(StringIO()), options=self.options)