Use the same python version to run storlet agents
This patch refactors package installations in storlets devstack plugin, so that we can run storlets agents with the same python version as the one we have in host to run storlets middleware. This patch also bumps up openjdk version from 8 to 11, and python3 version from 3.5 to 3.6, so that we use the latest versin provided in distro. Change-Id: I6cb87976ff4fa6caaa58cf555685f24d95f214a8
This commit is contained in:
Takashi Kajinami
parent
88b0ae777c
commit
f5815f065a
|
|
@ -75,25 +75,25 @@ TMP_REGISTRY_PREFIX=/tmp/registry
|
|||
# Functions
|
||||
# ---------
|
||||
|
||||
_storlets_swift_start() {
|
||||
function _storlets_swift_start {
|
||||
swift-init --run-dir=${SWIFT_DATA_DIR}/run all start || true
|
||||
}
|
||||
|
||||
_storlets_swift_stop() {
|
||||
function _storlets_swift_stop {
|
||||
swift-init --run-dir=${SWIFT_DATA_DIR}/run all stop || true
|
||||
}
|
||||
|
||||
_storlets_swift_restart() {
|
||||
function _storlets_swift_restart {
|
||||
swift-init --run-dir=${SWIFT_DATA_DIR}/run all restart || true
|
||||
}
|
||||
|
||||
_export_os_vars() {
|
||||
function _export_os_vars {
|
||||
export OS_IDENTITY_API_VERSION=3
|
||||
export OS_AUTH_URL="http://$KEYSTONE_IP/identity/v3"
|
||||
export OS_REGION_NAME=RegionOne
|
||||
}
|
||||
|
||||
_export_keystone_os_vars() {
|
||||
function _export_keystone_os_vars {
|
||||
_export_os_vars
|
||||
export OS_USERNAME=$ADMIN_USER
|
||||
export OS_USER_DOMAIN_ID=$STORLETS_DEFAULT_USER_DOMAIN_ID
|
||||
|
|
@ -102,7 +102,7 @@ _export_keystone_os_vars() {
|
|||
export OS_PROJECT_DOMAIN_ID=$STORLETS_DEFAULT_PROJECT_DOMAIN_ID
|
||||
}
|
||||
|
||||
_export_swift_os_vars() {
|
||||
function _export_swift_os_vars {
|
||||
_export_os_vars
|
||||
export OS_USERNAME=$SWIFT_DEFAULT_USER
|
||||
export OS_USER_DOMAIN_ID=$STORLETS_DEFAULT_USER_DOMAIN_ID
|
||||
|
|
@ -111,7 +111,7 @@ _export_swift_os_vars() {
|
|||
export OS_PROJECT_DOMAIN_ID=$STORLETS_DEFAULT_PROJECT_DOMAIN_ID
|
||||
}
|
||||
|
||||
configure_swift_and_keystone_for_storlets() {
|
||||
function configure_swift_and_keystone_for_storlets {
|
||||
# Add project and users to Keystone
|
||||
_export_keystone_os_vars
|
||||
project_test_created=$(openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | wc -l)
|
||||
|
|
@ -149,7 +149,7 @@ configure_swift_and_keystone_for_storlets() {
|
|||
swift post $STORLETS_LOG_CONTAIER_NAME
|
||||
}
|
||||
|
||||
_install_docker() {
|
||||
function _install_docker {
|
||||
# TODO: Add other dirstors.
|
||||
# This one is geared towards Ubuntu
|
||||
# See other projects that install docker
|
||||
|
|
@ -190,47 +190,50 @@ _install_docker() {
|
|||
fi
|
||||
}
|
||||
|
||||
prepare_storlets_install() {
|
||||
function prepare_storlets_install {
|
||||
sudo mkdir -p "$STORLETS_DOCKER_DEVICE"/docker
|
||||
sudo chmod 777 $STORLETS_DOCKER_DEVICE
|
||||
_install_docker
|
||||
sudo add-apt-repository -y ppa:openjdk-r/ppa
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y openjdk-8-jdk-headless
|
||||
sudo apt-get install -y ant
|
||||
sudo apt-get install -y python
|
||||
sudo apt-get install -y python-setuptools
|
||||
sudo apt-get install -y python3.5
|
||||
sudo apt-get install -y python3-setuptools
|
||||
|
||||
if is_ubuntu; then
|
||||
install_package openjdk-11-jdk-headless ant
|
||||
else
|
||||
die $LINENO "Unsupported distro"
|
||||
fi
|
||||
|
||||
if python3_enabled; then
|
||||
install_python3
|
||||
fi
|
||||
}
|
||||
|
||||
_generate_jre_dockerfile() {
|
||||
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/${STORLETS_DOCKER_BASE_IMG_NAME}_jre8/Dockerfile
|
||||
function _generate_jre_dockerfile {
|
||||
local PYTHON_PACKAGES='python2.7 python-dev python3.6 python3.6-dev'
|
||||
if python3_enabled; then
|
||||
PYTHON_PACKAGES="python2.7 python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev"
|
||||
fi
|
||||
|
||||
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/${STORLETS_DOCKER_BASE_IMG_NAME}_jre11/Dockerfile
|
||||
FROM $STORLETS_DOCKER_BASE_IMG
|
||||
MAINTAINER root
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install python -y && \
|
||||
apt-get install python3.5 -y && \
|
||||
apt-get install git -y && \
|
||||
apt-get update && \
|
||||
apt-get install openjdk-8-jre-headless -y && \
|
||||
apt-get install ${PYTHON_PACKAGES} openjdk-11-jre-headless -y && \
|
||||
apt-get clean
|
||||
EOF
|
||||
}
|
||||
|
||||
create_base_jre_image() {
|
||||
function create_base_jre_image {
|
||||
echo "Create base jre image"
|
||||
docker pull $STORLETS_DOCKER_BASE_IMG
|
||||
mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8
|
||||
mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11
|
||||
_generate_jre_dockerfile
|
||||
cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8
|
||||
docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8 .
|
||||
cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11
|
||||
docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11 .
|
||||
cd -
|
||||
}
|
||||
|
||||
_generate_logback_xml() {
|
||||
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets/logback.xml
|
||||
function _generate_logback_xml {
|
||||
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets/logback.xml
|
||||
<configuration>
|
||||
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>/tmp/SDaemon.log</file>
|
||||
|
|
@ -257,9 +260,9 @@ _generate_logback_xml() {
|
|||
EOF
|
||||
}
|
||||
|
||||
_generate_jre_storlet_dockerfile() {
|
||||
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets/Dockerfile
|
||||
FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8
|
||||
function _generate_jre_storlet_dockerfile {
|
||||
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets/Dockerfile
|
||||
FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11
|
||||
MAINTAINER root
|
||||
RUN [ "groupadd", "-g", "$STORLETS_DOCKER_SWIFT_GROUP_ID", "swift" ]
|
||||
RUN [ "useradd", "-u" , "$STORLETS_DOCKER_SWIFT_USER_ID", "-g", "$STORLETS_DOCKER_SWIFT_GROUP_ID", "swift" ]
|
||||
|
|
@ -275,25 +278,29 @@ ENTRYPOINT ["/usr/local/libexec/storlets/init_container.sh"]
|
|||
EOF
|
||||
}
|
||||
|
||||
create_storlet_engine_image() {
|
||||
function create_storlet_engine_image {
|
||||
echo "Create Storlet engine image"
|
||||
mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets
|
||||
mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets
|
||||
_generate_logback_xml
|
||||
_generate_jre_storlet_dockerfile
|
||||
cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets
|
||||
docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8_storlets .
|
||||
cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets
|
||||
docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11_storlets .
|
||||
cd -
|
||||
}
|
||||
|
||||
install_storlets_code() {
|
||||
function install_storlets_code {
|
||||
echo "Installing storlets"
|
||||
cd $REPO_DIR
|
||||
sudo ./install_libs.sh
|
||||
sudo pip install -r requirements.txt
|
||||
sudo python setup.py install
|
||||
sudo pip3 install -r requirements.txt
|
||||
sudo python3 setup.py install
|
||||
sudo chown -R ${STORLETS_SWIFT_RUNTIME_USER} storlets.egg-info*
|
||||
pip_install .
|
||||
|
||||
# Also install code to library directory so that we can import them
|
||||
# from docker container.
|
||||
sudo mkdir -p -m 755 /usr/local/lib/storlets/python
|
||||
pip_install . -t /usr/local/lib/storlets/python --no-compile
|
||||
for bin_file in storlets-daemon storlets-daemon-factory ; do
|
||||
sudo cp `which ${bin_file}` /usr/local/libexec/storlets/
|
||||
done
|
||||
|
||||
sudo mkdir -p $STORLETS_DOCKER_DEVICE/scripts
|
||||
sudo chown "$STORLETS_SWIFT_RUNTIME_USER":"$STORLETS_SWIFT_RUNTIME_GROUP" "$STORLETS_DOCKER_DEVICE"/scripts
|
||||
|
|
@ -305,7 +312,7 @@ install_storlets_code() {
|
|||
cd -
|
||||
}
|
||||
|
||||
_generate_swift_middleware_conf() {
|
||||
function _generate_swift_middleware_conf {
|
||||
cat <<EOF > /tmp/swift_middleware_conf
|
||||
[proxy-confs]
|
||||
proxy_server_conf_file = /etc/swift/proxy-server.conf
|
||||
|
|
@ -327,7 +334,7 @@ storlet_proxy_execution = $STORLETS_PROXY_EXECUTION_ONLY
|
|||
EOF
|
||||
}
|
||||
|
||||
_generate_storlet-docker-gateway() {
|
||||
function _generate_storlet-docker-gateway {
|
||||
cat <<EOF > /tmp/storlet-docker-gateway.conf
|
||||
[DEFAULT]
|
||||
storlet_logcontainer = $STORLETS_LOG_CONTAIER_NAME
|
||||
|
|
@ -341,14 +348,14 @@ storlet_timeout = $STORLETS_RUNTIME_TIMEOUT
|
|||
EOF
|
||||
}
|
||||
|
||||
_generate_default_tenant_dockerfile() {
|
||||
function _generate_default_tenant_dockerfile {
|
||||
cat <<EOF > ${TMP_REGISTRY_PREFIX}/repositories/"$SWIFT_DEFAULT_PROJECT_ID"/Dockerfile
|
||||
FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8_storlets
|
||||
FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11_storlets
|
||||
MAINTAINER root
|
||||
EOF
|
||||
}
|
||||
|
||||
create_default_tenant_image() {
|
||||
function create_default_tenant_image {
|
||||
SWIFT_DEFAULT_PROJECT_ID=`openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | awk '{ print $2 }'`
|
||||
mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/$SWIFT_DEFAULT_PROJECT_ID
|
||||
_generate_default_tenant_dockerfile
|
||||
|
|
@ -357,7 +364,7 @@ create_default_tenant_image() {
|
|||
cd -
|
||||
}
|
||||
|
||||
create_test_config_file() {
|
||||
function create_test_config_file {
|
||||
testfile=${REPO_DIR}/test.conf
|
||||
iniset ${testfile} general keystone_default_domain $STORLETS_DEFAULT_PROJECT_DOMAIN_ID
|
||||
iniset ${testfile} general keystone_public_url $KEYSTONE_PUBLIC_URL
|
||||
|
|
@ -369,8 +376,7 @@ create_test_config_file() {
|
|||
iniset ${testfile} general region
|
||||
}
|
||||
|
||||
|
||||
install_storlets() {
|
||||
function install_storlets {
|
||||
echo "Install storlets dependencies"
|
||||
prepare_storlets_install
|
||||
|
||||
|
|
@ -392,7 +398,7 @@ install_storlets() {
|
|||
_storlets_swift_restart
|
||||
}
|
||||
|
||||
uninstall_storlets() {
|
||||
function uninstall_storlets {
|
||||
sudo service docker stop
|
||||
sudo sed -r 's#^.*DOCKER_OPTS=.*$#DOCKER_OPTS="--debug --storage-opt dm.override_udev_sync_check=true"#' /etc/default/docker
|
||||
|
||||
|
|
|
|||
|
|
@ -119,10 +119,7 @@ We need the following for the Java parts
|
|||
|
||||
::
|
||||
|
||||
sudo add-apt-repository ppa:openjdk-r/ppa
|
||||
sudo apt-get update
|
||||
sudo apt-get install openjdk-8-jdk
|
||||
sudo apt-get install ant
|
||||
sudo apt-get install openjdk-11-jdk ant
|
||||
|
||||
We need the following for Docker
|
||||
|
||||
|
|
@ -160,9 +157,9 @@ Step 2: Create a Docker image with Java
|
|||
|
||||
::
|
||||
|
||||
mkdir -p $HOME/docker_repos/ubuntu_18.04_jre8
|
||||
mkdir -p $HOME/docker_repos/ubuntu_18.04_jre11
|
||||
|
||||
Create the file: $HOME/docker_repos/ubuntu_18.04_jre8/Dockerfile
|
||||
Create the file: $HOME/docker_repos/ubuntu_18.04_jre11/Dockerfile
|
||||
with the following content:
|
||||
|
||||
::
|
||||
|
|
@ -175,17 +172,15 @@ with the following content:
|
|||
RUN apt-get update && \
|
||||
apt-get install python && \
|
||||
apt-get install software-properties-common && \
|
||||
add-apt-repository ppa:openjdk-r/ppa && \
|
||||
apt-get update && \
|
||||
apt-get install openjdk-8-jre && \
|
||||
apt-get install openjdk-11-jre-headless ant && \
|
||||
apt-get clean
|
||||
|
||||
Build the image
|
||||
|
||||
::
|
||||
|
||||
cd $HOME/docker_repos/ubuntu_18.04_jre8
|
||||
sudo docker build -q -t ubuntu_18.04_jre8 .
|
||||
cd $HOME/docker_repos/ubuntu_18.04_jre11
|
||||
sudo docker build -q -t ubuntu_18.04_jre11 .
|
||||
cd -
|
||||
|
||||
|
||||
|
|
@ -193,16 +188,16 @@ Step 3: Augment the above created image with the storlets stuff
|
|||
|
||||
::
|
||||
|
||||
mkdir -p $HOME/docker_repos/ubuntu_18.04_jre8_storlets
|
||||
mkdir -p $HOME/docker_repos/ubuntu_18.04_jre11_storlets
|
||||
cp $HOME/storlets/install/storlets/roles/docker_storlet_engine_image/files/logback.xml .
|
||||
cd -
|
||||
|
||||
Create the file: $HOME/docker_repos/ubuntu_18.04_jre8_storlets/Dockerfile
|
||||
Create the file: $HOME/docker_repos/ubuntu_18.04_jre11_storlets/Dockerfile
|
||||
with the following content:
|
||||
|
||||
::
|
||||
|
||||
FROM ubuntu_18.04_jre8
|
||||
FROM ubuntu_18.04_jre11
|
||||
|
||||
MAINTAINER root
|
||||
|
||||
|
|
@ -222,8 +217,8 @@ Build the image
|
|||
|
||||
::
|
||||
|
||||
cd $HOME/docker_repos/ubuntu_18.04_jre8_storlets
|
||||
sudo docker build -q -t ubuntu_18.04_jre8_storlets .
|
||||
cd $HOME/docker_repos/ubuntu_18.04_jre11_storlets
|
||||
sudo docker build -q -t ubuntu_18.04_jre11_storlets .
|
||||
cd -
|
||||
|
||||
Step 4: Create a tenant specific image. The engine looks for images
|
||||
|
|
@ -242,12 +237,12 @@ The response from the above contains the account line, e.g.:
|
|||
|
||||
The account id is the number following the 'AUTH\_' prefix.
|
||||
|
||||
Next create the file $HOME/docker_repos/ubuntu_18.04_jre8_storlets_<account id>/Dockerfile
|
||||
Next create the file $HOME/docker_repos/ubuntu_18.04_jre11_storlets_<account id>/Dockerfile
|
||||
with the following content:
|
||||
|
||||
::
|
||||
|
||||
FROM ubuntu_18.04_jre8_storlets
|
||||
FROM ubuntu_18.04_jre11_storlets
|
||||
MAINTAINER root
|
||||
|
||||
|
||||
|
|
@ -255,7 +250,7 @@ Build the image
|
|||
|
||||
::
|
||||
|
||||
cd $HOME/docker_repos/ubuntu_18.04_jre8_storlets_<account id>
|
||||
cd $HOME/docker_repos/ubuntu_18.04_jre11_storlets_<account id>
|
||||
sudo docker build -q -t <account id> .
|
||||
cd -
|
||||
|
||||
|
|
|
|||
|
|
@ -23,14 +23,5 @@ ant install
|
|||
# Install container side scripts
|
||||
SRC=bin
|
||||
DST=/usr/local/libexec/storlets
|
||||
mkdir -p $DST
|
||||
chmod 755 $DST
|
||||
mkdir -p -m 755 $DST
|
||||
cp "$SRC/init_container.sh" $DST
|
||||
cp "$SRC/storlets-daemon" $DST
|
||||
cp "$SRC/storlets-daemon-factory" $DST
|
||||
|
||||
# Install python library to be mouted by containers
|
||||
DST=/usr/local/lib/storlets/python
|
||||
mkdir -p $DST
|
||||
chmod 755 $DST
|
||||
pip install . -t $DST --no-compile
|
||||
|
|
|
|||
|
|
@ -8,3 +8,5 @@
|
|||
./s2aio.sh install
|
||||
executable: /bin/bash
|
||||
chdir: 'src/{{ zuul.project.canonical_name }}'
|
||||
environment:
|
||||
USE_PYTHON3: "False"
|
||||
|
|
|
|||
1
s2aio.sh
1
s2aio.sh
|
|
@ -37,6 +37,7 @@ _prepare_devstack_env() {
|
|||
cp devstack/localrc.sample $DEVSTACK_DIR/localrc
|
||||
fi
|
||||
|
||||
source $DEVSTACK_DIR/stackrc
|
||||
source $DEVSTACK_DIR/functions
|
||||
source $DEVSTACK_DIR/functions-common
|
||||
source $DEVSTACK_DIR/lib/swift
|
||||
|
|
|
|||
|
|
@ -26,10 +26,6 @@ skip_changelog = True
|
|||
[files]
|
||||
packages =
|
||||
storlets
|
||||
scripts =
|
||||
bin/storlets-daemon
|
||||
bin/storlets-daemon-factory
|
||||
bin/sbus
|
||||
|
||||
[entry_points]
|
||||
paste.filter_factory =
|
||||
|
|
@ -39,6 +35,11 @@ storlets.gateways =
|
|||
stub = storlets.gateway.gateways.stub:StorletGatewayStub
|
||||
docker = storlets.gateway.gateways.docker:StorletGatewayDocker
|
||||
|
||||
console_scripts =
|
||||
sbus = storlets.sbus.cli:main
|
||||
storlets-daemon = storlets.agent.daemon.server:main
|
||||
storlets-daemon-factory = storlets.agent.daemon_factory.server:main
|
||||
|
||||
[build_sphinx]
|
||||
source-dir = doc/source
|
||||
build-dir = doc/build
|
||||
|
|
|
|||
|
|
@ -28,20 +28,15 @@
|
|||
srcdir="src/main"
|
||||
destdir="bin"
|
||||
classpath="../dependencies/json_simple-1.1.jar"
|
||||
includeantruntime="false"/>
|
||||
includeantruntime="false"
|
||||
nativeheaderdir='.'/>
|
||||
</target>
|
||||
|
||||
<target name="h" depends="java">
|
||||
<javah destdir="." force="yes" classpath="bin">
|
||||
<class name="org.openstack.storlet.sbus.SBusJNI" />
|
||||
</javah>
|
||||
</target>
|
||||
|
||||
<target name="so" depends="h">
|
||||
<target name="so" depends="java">
|
||||
<exec dir="." executable="gcc">
|
||||
<arg line="-shared -o bin/libjsbus.so -fPIC" />
|
||||
<arg line="-I/usr/lib/jvm/java-8-openjdk-amd64/include/" />
|
||||
<arg line="-I/usr/lib/jvm/java-8-openjdk-amd64/include/linux/" />
|
||||
<arg line="-I/usr/lib/jvm/java-11-openjdk-amd64/include/" />
|
||||
<arg line="-I/usr/lib/jvm/java-11-openjdk-amd64/include/linux/" />
|
||||
<arg line="-I../../c/sbus/" />
|
||||
<arg line="SBusJNI.c ../../c/sbus/sbus.c" />
|
||||
</exec>
|
||||
|
|
|
|||
|
|
@ -114,15 +114,15 @@ public class ServerSBusInDatagram {
|
|||
}
|
||||
|
||||
String strMD = msg.getMetadata();
|
||||
this.metadata = (HashMap<String, HashMap<String, String>>[])new HashMap[getNFiles()];
|
||||
this.metadata = (HashMap<String, HashMap<String, String>>[])new HashMap[getNFiles()];
|
||||
JSONArray jsonarray = (JSONArray)(new JSONParser().parse(strMD));
|
||||
Iterator it = jsonarray.iterator();
|
||||
Iterator<JSONObject> it = jsonarray.iterator();
|
||||
int i=0;
|
||||
while (it.hasNext()) {
|
||||
this.metadata[i] = new HashMap<String, HashMap<String, String>>();
|
||||
HashMap<String, String> storletsMetadata = new HashMap<String, String>();
|
||||
HashMap<String, String> storageMetadata = new HashMap<String, String>();
|
||||
JSONObject jsonobject = (JSONObject)it.next();
|
||||
JSONObject jsonobject = it.next();
|
||||
if (jsonobject.containsKey("storage")) {
|
||||
populateMetadata(storageMetadata, (JSONObject)jsonobject.get("storage"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ class SBusServer(object):
|
|||
handler = self.get_handler(command)
|
||||
resp = handler(dtg)
|
||||
except ValueError as err:
|
||||
self.logger.error(err.message)
|
||||
self.logger.exception('Failed to handle request')
|
||||
resp = CommandFailure(str(err))
|
||||
except CommandResponse as err:
|
||||
resp = err
|
||||
|
|
@ -140,7 +140,7 @@ class SBusServer(object):
|
|||
:param resp: CommandResponse instance
|
||||
"""
|
||||
try:
|
||||
outfile.write(resp.report_message)
|
||||
outfile.write(resp.report_message.encode('utf-8'))
|
||||
except IOError:
|
||||
self.logger.exception('Unable to return response to client')
|
||||
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@
|
|||
import logging
|
||||
from logging.handlers import SysLogHandler
|
||||
|
||||
# TODO(takashi): Make the following parameters configurable
|
||||
DEFAULT_PY2 = 2.7
|
||||
DEFAULT_PY3 = 3.5
|
||||
DEFAULT_PY3 = 3.6
|
||||
|
||||
|
||||
def get_logger(logger_name, log_level, container_id):
|
||||
|
|
|
|||
|
|
@ -14,12 +14,12 @@
|
|||
# limitations under the License.
|
||||
import argparse
|
||||
import errno
|
||||
import importlib
|
||||
import os
|
||||
import pwd
|
||||
import signal
|
||||
import sys
|
||||
import uuid
|
||||
import signal
|
||||
import importlib
|
||||
from storlets.sbus import SBus
|
||||
from storlets.agent.common.server import command_handler, EXIT_FAILURE, \
|
||||
CommandSuccess, CommandFailure, SBusServer
|
||||
|
|
@ -254,20 +254,22 @@ def main():
|
|||
# Initialize logger
|
||||
logger = get_logger("storlets-daemon", opts.log_level, opts.container_id)
|
||||
logger.debug("Storlet Daemon started")
|
||||
SBus.start_logger("DEBUG", container_id=opts.container_id)
|
||||
|
||||
# Impersonate the swift user
|
||||
pw = pwd.getpwnam('swift')
|
||||
os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid)
|
||||
os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid)
|
||||
|
||||
# create an instance of storlet daemon
|
||||
try:
|
||||
SBus.start_logger("DEBUG", container_id=opts.container_id)
|
||||
|
||||
# Impersonate the swift user
|
||||
pw = pwd.getpwnam('swift')
|
||||
os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid)
|
||||
os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid)
|
||||
|
||||
# create an instance of storlet daemon
|
||||
daemon = StorletDaemon(opts.storlet_name, opts.sbus_path,
|
||||
logger, opts.pool_size)
|
||||
except Exception as err:
|
||||
logger.error(err.message)
|
||||
return EXIT_FAILURE
|
||||
|
||||
# Start the main loop
|
||||
return daemon.main_loop()
|
||||
# Start the main loop
|
||||
sys.exit(daemon.main_loop())
|
||||
|
||||
except Exception:
|
||||
logger.error('Unhandled exception')
|
||||
sys.exit(EXIT_FAILURE)
|
||||
|
|
|
|||
|
|
@ -18,14 +18,15 @@ import os
|
|||
import pwd
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from storlets.sbus import SBus
|
||||
from storlets.sbus.client import SBusClient
|
||||
from storlets.sbus.client.exceptions import SBusClientException, \
|
||||
SBusClientSendError
|
||||
from storlets.agent.common.server import command_handler, CommandSuccess, \
|
||||
CommandFailure, SBusServer
|
||||
from storlets.agent.common.server import command_handler, EXIT_FAILURE, \
|
||||
CommandSuccess, CommandFailure, SBusServer
|
||||
from storlets.agent.common.utils import get_logger, DEFAULT_PY2, DEFAULT_PY3
|
||||
|
||||
|
||||
|
|
@ -107,6 +108,8 @@ class StorletDaemonFactory(SBusServer):
|
|||
if int(float(daemon_language_version)) == 3:
|
||||
daemon_language_version = DEFAULT_PY3
|
||||
else:
|
||||
# TODO(takashi): Switch default python version to 3 once we drop
|
||||
# python2 support.
|
||||
daemon_language_version = DEFAULT_PY2
|
||||
|
||||
python_interpreter = '/usr/bin/python%s' % daemon_language_version
|
||||
|
|
@ -521,15 +524,22 @@ def main():
|
|||
# Initialize logger
|
||||
logger = get_logger("daemon-factory", opts.log_level, opts.container_id)
|
||||
logger.debug("Daemon factory started")
|
||||
SBus.start_logger("DEBUG", container_id=opts.container_id)
|
||||
|
||||
# Impersonate the swift user
|
||||
pw = pwd.getpwnam('swift')
|
||||
os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid)
|
||||
os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid)
|
||||
try:
|
||||
SBus.start_logger("DEBUG", container_id=opts.container_id)
|
||||
|
||||
# create an instance of daemon_factory
|
||||
factory = StorletDaemonFactory(opts.sbus_path, logger, opts.container_id)
|
||||
# Impersonate the swift user
|
||||
pw = pwd.getpwnam('swift')
|
||||
os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid)
|
||||
os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid)
|
||||
|
||||
# Start the main loop
|
||||
return factory.main_loop()
|
||||
# create an instance of daemon_factory
|
||||
factory = StorletDaemonFactory(opts.sbus_path, logger,
|
||||
opts.container_id)
|
||||
|
||||
# Start the main loop
|
||||
sys.exit(factory.main_loop())
|
||||
|
||||
except Exception:
|
||||
logger.eception('Unhandled exception')
|
||||
sys.exit(EXIT_FAILURE)
|
||||
|
|
|
|||
|
|
@ -232,7 +232,8 @@ class RunTimeSandbox(object):
|
|||
self.logger = logger
|
||||
|
||||
self.default_docker_image_name = \
|
||||
conf.get('default_docker_image_name', 'ubuntu_18.04_jre8_storlets')
|
||||
conf.get('default_docker_image_name',
|
||||
'ubuntu_18.04_jre11_storlets')
|
||||
|
||||
def ping(self):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
from sys import exit
|
||||
import sys
|
||||
from storlets.sbus.client import SBusClient
|
||||
from storlets.sbus.client.exceptions import SBusClientException
|
||||
|
||||
|
|
@ -21,19 +21,20 @@ EXIT_SUCCESS = 0
|
|||
EXIT_ERROR = 1
|
||||
|
||||
|
||||
def main(argv):
|
||||
# TODO(takashi): Add more detailed help message
|
||||
def main():
|
||||
argv = sys.argv
|
||||
|
||||
# TODO(takashi): Add more detailed help message
|
||||
if len(argv) < 3:
|
||||
print('sbus <command> <pipe_path>')
|
||||
exit(EXIT_ERROR)
|
||||
sys.exit(EXIT_ERROR)
|
||||
|
||||
command = argv[1]
|
||||
pipe_path = argv[2]
|
||||
|
||||
if not os.path.exists(pipe_path):
|
||||
print('ERROR: Pipe file %s does not exist' % pipe_path)
|
||||
exit(EXIT_ERROR)
|
||||
sys.exit(EXIT_ERROR)
|
||||
|
||||
client = SBusClient(pipe_path)
|
||||
try:
|
||||
|
|
@ -45,19 +46,19 @@ def main(argv):
|
|||
resp = handler()
|
||||
except (AttributeError, NotImplementedError):
|
||||
print('ERROR: Command %s is not supported' % command)
|
||||
exit(EXIT_ERROR)
|
||||
sys.exit(EXIT_ERROR)
|
||||
except SBusClientException as err:
|
||||
print('ERROR: Failed to send sbus command %s to %s: %s'
|
||||
% (command, pipe_path, err))
|
||||
exit(EXIT_ERROR)
|
||||
sys.exit(EXIT_ERROR)
|
||||
except Exception as err:
|
||||
print('ERROR: Unknown error: %s' % err)
|
||||
exit(EXIT_ERROR)
|
||||
sys.exit(EXIT_ERROR)
|
||||
|
||||
print('Response: %s: %s' % (resp.status, resp.message))
|
||||
if resp.status:
|
||||
print('OK')
|
||||
exit(EXIT_SUCCESS)
|
||||
sys.exit(EXIT_SUCCESS)
|
||||
else:
|
||||
print('ERROR: Got error response')
|
||||
exit(EXIT_ERROR)
|
||||
sys.exit(EXIT_ERROR)
|
||||
|
|
|
|||
|
|
@ -62,7 +62,8 @@ class SBus(object):
|
|||
sbus_back_ = CDLL(SBus.SBUS_SO_NAME)
|
||||
|
||||
sbus_back_.sbus_start_logger.argtypes = [c_char_p, c_char_p]
|
||||
sbus_back_.sbus_start_logger(str_log_level, container_id)
|
||||
sbus_back_.sbus_start_logger(str_log_level.encode("utf-8"),
|
||||
container_id.encode("utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def stop_logger():
|
||||
|
|
|
|||
|
|
@ -195,9 +195,9 @@ class StorletProxyHandler(StorletBaseHandler):
|
|||
acl_string = '.r:%s' % self._build_acl_string(user_name, storlet_name)
|
||||
try:
|
||||
clean_acl('X-Container-Read', acl_string)
|
||||
except ValueError as e:
|
||||
except ValueError as err:
|
||||
msg = ('storlet ACL update request has invalid values %s'
|
||||
% e.message)
|
||||
% str(err))
|
||||
raise HTTPBadRequest(msg.encode('utf8'))
|
||||
|
||||
# Make sure the resulting acl permits a single entity
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
|
|||
|
||||
headers = c.head_object(self.url, self.token,
|
||||
self.container, 'gen_thumb_on_put.jpg')
|
||||
self.assertEqual('49032', headers['content-length'])
|
||||
self.assertLess(int(headers['content-length']), 1087318)
|
||||
self.assertEqual('thumbnail', headers['x-object-meta-name'])
|
||||
|
||||
def invoke_storlet_on_copy_from(self):
|
||||
|
|
@ -88,7 +88,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
|
|||
|
||||
headers = c.head_object(self.url, self.token,
|
||||
self.container, 'gen_thumb_on_copy.jpg')
|
||||
self.assertEqual('49032', headers['content-length'])
|
||||
self.assertLess(int(headers['content-length']), 1087318)
|
||||
self.assertEqual('thumbnail', headers['x-object-meta-name'])
|
||||
self.assertTrue('x-object-meta-x-timestamp' not in headers)
|
||||
self.assertTrue('x-timestamp' in headers)
|
||||
|
|
@ -109,7 +109,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest):
|
|||
|
||||
headers = c.head_object(self.url, self.token,
|
||||
self.container, 'gen_thumb_on_copy_.jpg')
|
||||
self.assertEqual('49032', headers['content-length'])
|
||||
self.assertLess(int(headers['content-length']), 1087318)
|
||||
self.assertEqual('thumbnail', headers['x-object-meta-name'])
|
||||
self.assertTrue('x-object-meta-x-timestamp' not in headers)
|
||||
self.assertTrue('x-timestamp' in headers)
|
||||
|
|
|
|||
Loading…
Reference in New Issue