diff --git a/integration/scripts/files/elements/ubuntu-redis/install.d/30-redis b/integration/scripts/files/elements/ubuntu-redis/install.d/30-redis index 84b091f16d..f9ab9a0a00 100755 --- a/integration/scripts/files/elements/ubuntu-redis/install.d/30-redis +++ b/integration/scripts/files/elements/ubuntu-redis/install.d/30-redis @@ -32,9 +32,75 @@ fi _EOF_ -add-apt-repository -y ppa:chris-lea/redis-server -apt-get -y update -apt-get --allow-unauthenticated install -y redis-server +# Install Redis from scratch +ARCH_SAV=$ARCH +unset ARCH +apt-get update +apt-get --allow-unauthenticated install -y build-essential tcl curl +cd /tmp +REDIS_VERSION=3.2.6 +REDIS_NAME=redis-$REDIS_VERSION +REDIS_ARCHIVE=$REDIS_NAME.tar.gz +curl -O http://download.redis.io/releases/$REDIS_ARCHIVE +tar xzvf $REDIS_ARCHIVE +cd $REDIS_NAME +make distclean +make +make install +export ARCH=$ARCH_SAV + +adduser --system --group --no-create-home redis +# Create the data dir +REDIS_DATA_DIR=/var/lib/redis +REDIS_LOG_DIR=/var/log/redis +REDIS_LOG=$REDIS_LOG_DIR/redis.log +REDIS_RUN_DIR=/var/run/redis +REDIS_PID=$REDIS_RUN_DIR/redis-server.pid +mkdir -p $REDIS_DATA_DIR +chown redis:redis $REDIS_DATA_DIR +chmod 770 $REDIS_DATA_DIR +mkdir -p $REDIS_LOG_DIR +chown redis:redis $REDIS_LOG_DIR +chmod 775 $REDIS_LOG_DIR +mkdir -p $REDIS_RUN_DIR +chown redis:redis $REDIS_RUN_DIR +# Set up a proper conf file to start +REDIS_CONF_NAME=redis.conf +REDIS_CONF_DIR=/etc/redis +REDIS_CONF=$REDIS_CONF_DIR/$REDIS_CONF_NAME +mkdir $REDIS_CONF_DIR +cp /tmp/$REDIS_NAME/$REDIS_CONF_NAME $REDIS_CONF_DIR +chown redis:redis $REDIS_CONF +sed -i "s#dir .*#dir $REDIS_DATA_DIR#" $REDIS_CONF +sed -i "s#pidfile .*#pidfile $REDIS_PID#" $REDIS_CONF +sed -i "s#logfile .*#logfile $REDIS_LOG#" $REDIS_CONF +sed -i "s/supervised no/supervised systemd/" $REDIS_CONF +sed -i "s/daemonize no/daemonize yes/" $REDIS_CONF +sed -i "s/protected-mode yes/protected-mode no/" $REDIS_CONF + +cat > "/etc/systemd/system/redis-server.service" << _EOF_ +[Unit] +Description=Redis In-Memory Data Store +After=network.target + +[Service] +Type=forking +PIDFile=$REDIS_PID +User=redis +Group=redis + +Environment=statedir=$REDIS_RUN_DIR +PermissionsStartOnly=true +ExecStartPre=/bin/mkdir -p \${statedir} +ExecStartPre=/bin/chown -R redis:redis \${statedir} +ExecStart=/usr/local/bin/redis-server $REDIS_CONF +ExecReload=/bin/kill -USR2 \$MAINPID +ExecStop=/usr/local/bin/redis-cli shutdown +Restart=always + +[Install] +WantedBy=multi-user.target +_EOF_ cat > "/etc/default/redis-server" << _EOF_ # Call ulimit -n with this argument prior to invoking Redis itself. @@ -46,8 +112,3 @@ _EOF_ # Install Python driver for Redis ('redis-py'). pip2 install redis - -# By default, redis-py will attempt to use the HiredisParser if installed. -# Using Hiredis can provide up to a 10x speed improvement in parsing responses -# from the Redis server. -pip2 install hiredis diff --git a/integration/scripts/trovestack b/integration/scripts/trovestack index c9e43b80ba..405a04956c 100755 --- a/integration/scripts/trovestack +++ b/integration/scripts/trovestack @@ -566,8 +566,8 @@ function cmd_set_datastore() { PACKAGES=${PACKAGES:-"mongodb-org"} VERSION="3.2" elif [ "$DATASTORE_TYPE" == "redis" ]; then - PACKAGES=${PACKAGES:-"redis-server"} - VERSION="3.0" + PACKAGES=${PACKAGES:-""} + VERSION="3.2.6" elif [ "$DATASTORE_TYPE" == "cassandra" ]; then PACKAGES=${PACKAGES:-"cassandra"} VERSION="2.1.0" diff --git a/trove/guestagent/datastore/experimental/redis/service.py b/trove/guestagent/datastore/experimental/redis/service.py index 4b3d86a8d6..d3c747ac90 100644 --- a/trove/guestagent/datastore/experimental/redis/service.py +++ b/trove/guestagent/datastore/experimental/redis/service.py @@ -215,6 +215,8 @@ class RedisApp(object): self.configuration_manager.apply_system_override( {'daemonize': 'yes', + 'protected-mode': 'no', + 'supervised': 'systemd', 'pidfile': system.REDIS_PID_FILE, 'logfile': system.REDIS_LOG_FILE, 'dir': system.REDIS_DATA_DIR}) @@ -341,7 +343,7 @@ class RedisApp(object): slots = map(str, range(first_slot, last_slot + 1)) group_size = 200 while slots: - cmd = ([system.REDIS_CLI, 'cluster', 'addslots'] + cmd = (['redis-cli', 'cluster', 'addslots'] + slots[0:group_size]) out, err = utils.execute_with_timeout(*cmd, run_as_root=True, root_helper='sudo') diff --git a/trove/guestagent/datastore/experimental/redis/system.py b/trove/guestagent/datastore/experimental/redis/system.py index d8e4b3fba1..e10d38457a 100644 --- a/trove/guestagent/datastore/experimental/redis/system.py +++ b/trove/guestagent/datastore/experimental/redis/system.py @@ -28,9 +28,7 @@ REDIS_CONF_DIR = '/etc/redis' REDIS_DATA_DIR = '/var/lib/redis' REDIS_PORT = '6379' REDIS_INIT = '/etc/init/redis-server.conf' -REDIS_CLI = '/usr/bin/redis-cli' -REDIS_BIN = '/usr/bin/redis-server' -REDIS_PACKAGE = 'redis-server' +REDIS_PACKAGE = '' SERVICE_CANDIDATES = ['redis-server', 'redis'] OS = get_os() diff --git a/trove/templates/redis/config.template b/trove/templates/redis/config.template index 71c6a26d96..725700eb79 100644 --- a/trove/templates/redis/config.template +++ b/trove/templates/redis/config.template @@ -58,6 +58,25 @@ port 6379 # in order to get the desired effect. tcp-backlog 511 +# Protected mode is a layer of security protection, in order to avoid that +# Redis instances left open on the internet are accessed and exploited. +# +# When protected mode is on and if: +# +# 1) The server is not binding explicitly to a set of addresses using the +# "bind" directive. +# 2) No password is configured. +# +# The server only accepts connections from clients connecting from the +# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain +# sockets. +# +# By default protected mode is enabled. You should disable it only if +# you are sure you want clients from other hosts to connect to Redis +# even if no authentication is configured, nor a specific set of interfaces +# are explicitly listed using the "bind" directive. +protected-mode no + # By default Redis listens for connections from all the network interfaces # available on the server. It is possible to listen to just one or multiple # interfaces using the "bind" configuration directive, followed by one or @@ -96,6 +115,17 @@ timeout 0 # A reasonable value for this option is 60 seconds. tcp-keepalive 0 +# If you run Redis from upstart or systemd, Redis can interact with your +# supervision tree. Options: +# supervised no - no supervision interaction +# supervised upstart - signal upstart by putting Redis into SIGSTOP mode +# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET +# supervised auto - detect upstart or systemd method based on +# UPSTART_JOB or NOTIFY_SOCKET environment variables +# Note: these supervision methods only signal "process is ready." +# They do not enable continuous liveness pings back to your supervisor. +supervised systemd + # Specify the server verbosity level. # This can be one of: # debug (a lot of information, useful for development/testing) @@ -844,6 +874,37 @@ hash-max-ziplist-value 64 list-max-ziplist-entries 512 list-max-ziplist-value 64 +# Lists are also encoded in a special way to save a lot of space. +# The number of entries allowed per internal list node can be specified +# as a fixed maximum size or a maximum number of elements. +# For a fixed maximum size, use -5 through -1, meaning: +# -5: max size: 64 Kb <-- not recommended for normal workloads +# -4: max size: 32 Kb <-- not recommended +# -3: max size: 16 Kb <-- probably not recommended +# -2: max size: 8 Kb <-- good +# -1: max size: 4 Kb <-- good +# Positive numbers mean store up to _exactly_ that number of elements +# per list node. +# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), +# but if your use case is unique, adjust the settings as necessary. +list-max-ziplist-size -2 +# +# Lists may also be compressed. +# Compress depth is the number of quicklist ziplist nodes from *each* side of +# the list to *exclude* from compression. The head and tail of the list +# are always uncompressed for fast push/pop operations. Settings are: +# 0: disable all list compression +# 1: depth 1 means "don't start compressing until after 1 node into the list, +# going from either the head or tail" +# So: [head]->node->node->...->node->[tail] +# [head], [tail] will always be uncompressed; inner nodes will compress. +# 2: [head]->[next]->node->node->...->node->[prev]->[tail] +# 2 here means: don't compress head or head->next or tail->prev or tail, +# but compress all nodes between them. +# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] +# etc. +list-compress-depth 0 + # Sets have a special encoding in just one case: when a set is composed # of just strings that happen to be integers in radix 10 in the range # of 64 bit signed integers. @@ -951,4 +1012,4 @@ hz 10 # the file will be fsync-ed every 32 MB of data generated. This is useful # in order to commit the file to the disk more incrementally and avoid # big latency spikes. -aof-rewrite-incremental-fsync yes \ No newline at end of file +aof-rewrite-incremental-fsync yes diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py index 6bae63a1c6..2bc7775012 100644 --- a/trove/tests/int_tests.py +++ b/trove/tests/int_tests.py @@ -405,10 +405,10 @@ register( single=[common_groups, backup_groups, configuration_groups, ], - multi=[ - # cluster_actions_groups, - # cluster_negative_actions_groups, - replication_promote_groups, ] + multi=[replication_promote_groups, ] + # multi=[cluster_actions_groups, + # cluster_negative_actions_groups, + # replication_promote_groups, ] ) register( diff --git a/trove/tests/scenario/helpers/test_helper.py b/trove/tests/scenario/helpers/test_helper.py index dc4cfcd750..50736de561 100644 --- a/trove/tests/scenario/helpers/test_helper.py +++ b/trove/tests/scenario/helpers/test_helper.py @@ -39,18 +39,18 @@ class DataType(Enum): micro4 = 4 # very tiny amount of data, useful for testing replication # propagation, etc. - tiny = 3 + tiny = 5 # another tiny dataset (also for replication propagation) - tiny2 = 4 + tiny2 = 6 # a third tiny dataset (also for replication propagation) - tiny3 = 5 + tiny3 = 7 # a forth tiny dataset (for cluster propagation) - tiny4 = 6 + tiny4 = 8 # small amount of data (this can be added to each instance # after creation, for example). - small = 7 + small = 9 # large data, enough to make creating a backup take 20s or more. - large = 8 + large = 10 class TestHelper(object):