From 55d8598d206cdfbc7cc7c07fdf9ba2f5fa59e81d Mon Sep 17 00:00:00 2001 From: Peter Stachowski Date: Tue, 3 Jan 2017 13:57:06 -0800 Subject: [PATCH] Install Redis 3.2.6 by compilation The version of Redis installed by the current elements uses a ppa that is stalled at 3.0.7. This seems to have some issues that are fixed in later versions. The recommended method of installing Redis is to compile the code yourself, so the elements have been changed to do this. The version was bumped to 3.2.6 (latest stable) however the Redis cluster tests were not reenabled as there still seems to be an issue with them. New config options were added to the template; these may need to be updated to make sure all the new ones are there. Change-Id: I1a604b15ade815cdb51c5484d4285e504508a6c4 Partial-Bug: 1652964 --- .../elements/ubuntu-redis/install.d/30-redis | 77 +++++++++++++++++-- integration/scripts/trovestack | 4 +- .../datastore/experimental/redis/service.py | 4 +- .../datastore/experimental/redis/system.py | 4 +- trove/templates/redis/config.template | 63 ++++++++++++++- trove/tests/int_tests.py | 8 +- trove/tests/scenario/helpers/test_helper.py | 12 +-- 7 files changed, 147 insertions(+), 25 deletions(-) diff --git a/integration/scripts/files/elements/ubuntu-redis/install.d/30-redis b/integration/scripts/files/elements/ubuntu-redis/install.d/30-redis index 84b091f16d..f9ab9a0a00 100755 --- a/integration/scripts/files/elements/ubuntu-redis/install.d/30-redis +++ b/integration/scripts/files/elements/ubuntu-redis/install.d/30-redis @@ -32,9 +32,75 @@ fi _EOF_ -add-apt-repository -y ppa:chris-lea/redis-server -apt-get -y update -apt-get --allow-unauthenticated install -y redis-server +# Install Redis from scratch +ARCH_SAV=$ARCH +unset ARCH +apt-get update +apt-get --allow-unauthenticated install -y build-essential tcl curl +cd /tmp +REDIS_VERSION=3.2.6 +REDIS_NAME=redis-$REDIS_VERSION +REDIS_ARCHIVE=$REDIS_NAME.tar.gz +curl -O http://download.redis.io/releases/$REDIS_ARCHIVE +tar xzvf $REDIS_ARCHIVE +cd $REDIS_NAME +make distclean +make +make install +export ARCH=$ARCH_SAV + +adduser --system --group --no-create-home redis +# Create the data dir +REDIS_DATA_DIR=/var/lib/redis +REDIS_LOG_DIR=/var/log/redis +REDIS_LOG=$REDIS_LOG_DIR/redis.log +REDIS_RUN_DIR=/var/run/redis +REDIS_PID=$REDIS_RUN_DIR/redis-server.pid +mkdir -p $REDIS_DATA_DIR +chown redis:redis $REDIS_DATA_DIR +chmod 770 $REDIS_DATA_DIR +mkdir -p $REDIS_LOG_DIR +chown redis:redis $REDIS_LOG_DIR +chmod 775 $REDIS_LOG_DIR +mkdir -p $REDIS_RUN_DIR +chown redis:redis $REDIS_RUN_DIR +# Set up a proper conf file to start +REDIS_CONF_NAME=redis.conf +REDIS_CONF_DIR=/etc/redis +REDIS_CONF=$REDIS_CONF_DIR/$REDIS_CONF_NAME +mkdir $REDIS_CONF_DIR +cp /tmp/$REDIS_NAME/$REDIS_CONF_NAME $REDIS_CONF_DIR +chown redis:redis $REDIS_CONF +sed -i "s#dir .*#dir $REDIS_DATA_DIR#" $REDIS_CONF +sed -i "s#pidfile .*#pidfile $REDIS_PID#" $REDIS_CONF +sed -i "s#logfile .*#logfile $REDIS_LOG#" $REDIS_CONF +sed -i "s/supervised no/supervised systemd/" $REDIS_CONF +sed -i "s/daemonize no/daemonize yes/" $REDIS_CONF +sed -i "s/protected-mode yes/protected-mode no/" $REDIS_CONF + +cat > "/etc/systemd/system/redis-server.service" << _EOF_ +[Unit] +Description=Redis In-Memory Data Store +After=network.target + +[Service] +Type=forking +PIDFile=$REDIS_PID +User=redis +Group=redis + +Environment=statedir=$REDIS_RUN_DIR +PermissionsStartOnly=true +ExecStartPre=/bin/mkdir -p \${statedir} +ExecStartPre=/bin/chown -R redis:redis \${statedir} +ExecStart=/usr/local/bin/redis-server $REDIS_CONF +ExecReload=/bin/kill -USR2 \$MAINPID +ExecStop=/usr/local/bin/redis-cli shutdown +Restart=always + +[Install] +WantedBy=multi-user.target +_EOF_ cat > "/etc/default/redis-server" << _EOF_ # Call ulimit -n with this argument prior to invoking Redis itself. @@ -46,8 +112,3 @@ _EOF_ # Install Python driver for Redis ('redis-py'). pip2 install redis - -# By default, redis-py will attempt to use the HiredisParser if installed. -# Using Hiredis can provide up to a 10x speed improvement in parsing responses -# from the Redis server. -pip2 install hiredis diff --git a/integration/scripts/trovestack b/integration/scripts/trovestack index c9e43b80ba..405a04956c 100755 --- a/integration/scripts/trovestack +++ b/integration/scripts/trovestack @@ -566,8 +566,8 @@ function cmd_set_datastore() { PACKAGES=${PACKAGES:-"mongodb-org"} VERSION="3.2" elif [ "$DATASTORE_TYPE" == "redis" ]; then - PACKAGES=${PACKAGES:-"redis-server"} - VERSION="3.0" + PACKAGES=${PACKAGES:-""} + VERSION="3.2.6" elif [ "$DATASTORE_TYPE" == "cassandra" ]; then PACKAGES=${PACKAGES:-"cassandra"} VERSION="2.1.0" diff --git a/trove/guestagent/datastore/experimental/redis/service.py b/trove/guestagent/datastore/experimental/redis/service.py index 4b3d86a8d6..d3c747ac90 100644 --- a/trove/guestagent/datastore/experimental/redis/service.py +++ b/trove/guestagent/datastore/experimental/redis/service.py @@ -215,6 +215,8 @@ class RedisApp(object): self.configuration_manager.apply_system_override( {'daemonize': 'yes', + 'protected-mode': 'no', + 'supervised': 'systemd', 'pidfile': system.REDIS_PID_FILE, 'logfile': system.REDIS_LOG_FILE, 'dir': system.REDIS_DATA_DIR}) @@ -341,7 +343,7 @@ class RedisApp(object): slots = map(str, range(first_slot, last_slot + 1)) group_size = 200 while slots: - cmd = ([system.REDIS_CLI, 'cluster', 'addslots'] + cmd = (['redis-cli', 'cluster', 'addslots'] + slots[0:group_size]) out, err = utils.execute_with_timeout(*cmd, run_as_root=True, root_helper='sudo') diff --git a/trove/guestagent/datastore/experimental/redis/system.py b/trove/guestagent/datastore/experimental/redis/system.py index d8e4b3fba1..e10d38457a 100644 --- a/trove/guestagent/datastore/experimental/redis/system.py +++ b/trove/guestagent/datastore/experimental/redis/system.py @@ -28,9 +28,7 @@ REDIS_CONF_DIR = '/etc/redis' REDIS_DATA_DIR = '/var/lib/redis' REDIS_PORT = '6379' REDIS_INIT = '/etc/init/redis-server.conf' -REDIS_CLI = '/usr/bin/redis-cli' -REDIS_BIN = '/usr/bin/redis-server' -REDIS_PACKAGE = 'redis-server' +REDIS_PACKAGE = '' SERVICE_CANDIDATES = ['redis-server', 'redis'] OS = get_os() diff --git a/trove/templates/redis/config.template b/trove/templates/redis/config.template index 71c6a26d96..725700eb79 100644 --- a/trove/templates/redis/config.template +++ b/trove/templates/redis/config.template @@ -58,6 +58,25 @@ port 6379 # in order to get the desired effect. tcp-backlog 511 +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode no + # By default Redis listens for connections from all the network interfaces # available on the server. It is possible to listen to just one or multiple # interfaces using the "bind" configuration directive, followed by one or @@ -96,6 +115,17 @@ timeout 0 # A reasonable value for this option is 60 seconds. tcp-keepalive 0 +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous liveness pings back to your supervisor. +supervised systemd + # Specify the server verbosity level. # This can be one of: # debug (a lot of information, useful for development/testing) @@ -844,6 +874,37 @@ hash-max-ziplist-value 64 list-max-ziplist-entries 512 list-max-ziplist-value 64 +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 +# +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 + # Sets have a special encoding in just one case: when a set is composed # of just strings that happen to be integers in radix 10 in the range # of 64 bit signed integers. @@ -951,4 +1012,4 @@ hz 10 # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid # big latency spikes. -aof-rewrite-incremental-fsync yes \ No newline at end of file +aof-rewrite-incremental-fsync yes diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py index 6bae63a1c6..2bc7775012 100644 --- a/trove/tests/int_tests.py +++ b/trove/tests/int_tests.py @@ -405,10 +405,10 @@ register( single=[common_groups, backup_groups, configuration_groups, ], - multi=[ - # cluster_actions_groups, - # cluster_negative_actions_groups, - replication_promote_groups, ] + multi=[replication_promote_groups, ] + # multi=[cluster_actions_groups, + # cluster_negative_actions_groups, + # replication_promote_groups, ] ) register( diff --git a/trove/tests/scenario/helpers/test_helper.py b/trove/tests/scenario/helpers/test_helper.py index dc4cfcd750..50736de561 100644 --- a/trove/tests/scenario/helpers/test_helper.py +++ b/trove/tests/scenario/helpers/test_helper.py @@ -39,18 +39,18 @@ class DataType(Enum): micro4 = 4 # very tiny amount of data, useful for testing replication # propagation, etc. - tiny = 3 + tiny = 5 # another tiny dataset (also for replication propagation) - tiny2 = 4 + tiny2 = 6 # a third tiny dataset (also for replication propagation) - tiny3 = 5 + tiny3 = 7 # a forth tiny dataset (for cluster propagation) - tiny4 = 6 + tiny4 = 8 # small amount of data (this can be added to each instance # after creation, for example). - small = 7 + small = 9 # large data, enough to make creating a backup take 20s or more. - large = 8 + large = 10 class TestHelper(object):