From 33a96ffc2609d8477542002e56515a5026c2c48d Mon Sep 17 00:00:00 2001 From: Davanum Srinivas Date: Mon, 2 Nov 2015 17:23:39 -0500 Subject: [PATCH] Zookeeper for DLM scenarios In Tokyo, there was a cross project session on distributed key locking: https://etherpad.openstack.org/p/mitaka-cross-project-dlm In support of the discussion there, we'll need support for a zookeeper service in Devstack and ability to use libraries like Tooz for DLM functionality. In this review, we pick up some configuration files from monasca-api and copy the lib/template to implement the zookeeper lifecycle. Those services that need zookeeper need to add "zookeeper" in ENABLED_SERVICES. Change-Id: Icef26e5cdaa930a581e27d330e47706776a7f98f --- files/debs/zookeeper | 1 + files/rpms/zookeeper | 1 + files/zookeeper/environment | 36 +++++++++++++ files/zookeeper/log4j.properties | 69 +++++++++++++++++++++++++ files/zookeeper/myid | 1 + files/zookeeper/zoo.cfg | 74 +++++++++++++++++++++++++++ lib/zookeeper | 86 ++++++++++++++++++++++++++++++++ stack.sh | 15 ++++++ stackrc | 2 +- unstack.sh | 5 ++ 10 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 files/debs/zookeeper create mode 100644 files/rpms/zookeeper create mode 100644 files/zookeeper/environment create mode 100644 files/zookeeper/log4j.properties create mode 100644 files/zookeeper/myid create mode 100644 files/zookeeper/zoo.cfg create mode 100644 lib/zookeeper diff --git a/files/debs/zookeeper b/files/debs/zookeeper new file mode 100644 index 0000000000..66227f7e31 --- /dev/null +++ b/files/debs/zookeeper @@ -0,0 +1 @@ +zookeeperd \ No newline at end of file diff --git a/files/rpms/zookeeper b/files/rpms/zookeeper new file mode 100644 index 0000000000..c0d1c3066d --- /dev/null +++ b/files/rpms/zookeeper @@ -0,0 +1 @@ +zookeeper \ No newline at end of file diff --git a/files/zookeeper/environment b/files/zookeeper/environment new file mode 100644 index 0000000000..afa2d2f89f --- /dev/null +++ b/files/zookeeper/environment @@ -0,0 +1,36 @@ +# +# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#    http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Modified from http://packages.ubuntu.com/saucy/zookeeperd +NAME=zookeeper +ZOOCFGDIR=/etc/zookeeper/conf + +# seems, that log4j requires the log4j.properties file to be in the classpath +CLASSPATH="$ZOOCFGDIR:/usr/share/java/jline.jar:/usr/share/java/log4j-1.2.jar:/usr/share/java/xercesImpl.jar:/usr/share/java/xmlParserAPIs.jar:/usr/share/java/netty.jar:/usr/share/java/slf4j-api.jar:/usr/share/java/slf4j-log4j12.jar:/usr/share/java/zookeeper.jar" + +ZOOCFG="$ZOOCFGDIR/zoo.cfg" +ZOO_LOG_DIR=/var/log/zookeeper +USER=$NAME +GROUP=$NAME +PIDDIR=/var/run/$NAME +PIDFILE=$PIDDIR/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +JAVA=/usr/bin/java +ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain" +ZOO_LOG4J_PROP="INFO,ROLLINGFILE" +JMXLOCALONLY=false +JAVA_OPTS="" diff --git a/files/zookeeper/log4j.properties b/files/zookeeper/log4j.properties new file mode 100644 index 0000000000..6c45a4aad9 --- /dev/null +++ b/files/zookeeper/log4j.properties @@ -0,0 +1,69 @@ +# +# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#    http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# From http://packages.ubuntu.com/saucy/zookeeperd + +# ZooKeeper Logging Configuration +# + +# Format is " (, )+ + +log4j.rootLogger=${zookeeper.root.logger} + +# Example: console appender only +# log4j.rootLogger=INFO, CONSOLE + +# Example with rolling log file +#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE + +# Example with rolling log file and tracing +#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE + +# +# Log INFO level and above messages to the console +# +log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender +log4j.appender.CONSOLE.Threshold=INFO +log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout +log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + +# +# Add ROLLINGFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender +log4j.appender.ROLLINGFILE.Threshold=WARN +log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/zookeeper.log + +# Max log file size of 10MB +log4j.appender.ROLLINGFILE.MaxFileSize=10MB +# uncomment the next line to limit number of backup files +#log4j.appender.ROLLINGFILE.MaxBackupIndex=10 + +log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n + + +# +# Add TRACEFILE to rootLogger to get log file output +# Log DEBUG level and above messages to a log file +log4j.appender.TRACEFILE=org.apache.log4j.FileAppender +log4j.appender.TRACEFILE.Threshold=TRACE +log4j.appender.TRACEFILE.File=${zookeeper.log.dir}/zookeeper_trace.log + +log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout +### Notice we are including log4j's NDC here (%x) +log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n diff --git a/files/zookeeper/myid b/files/zookeeper/myid new file mode 100644 index 0000000000..c227083464 --- /dev/null +++ b/files/zookeeper/myid @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/files/zookeeper/zoo.cfg b/files/zookeeper/zoo.cfg new file mode 100644 index 0000000000..b8f55827e3 --- /dev/null +++ b/files/zookeeper/zoo.cfg @@ -0,0 +1,74 @@ +# +# (C) Copyright 2015 Hewlett Packard Enterprise Development Company LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#    http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html + +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 +# the directory where the snapshot is stored. +dataDir=/var/lib/zookeeper +# Place the dataLogDir to a separate physical disc for better performance +# dataLogDir=/disk2/zookeeper + +# the port at which the clients will connect +clientPort=2181 + +# Maximum number of clients that can connect from one client +maxClientCnxns=60 + +# specify all zookeeper servers +# The fist port is used by followers to connect to the leader +# The second one is used for leader election + +server.0=127.0.0.1:2888:3888 + +# To avoid seeks ZooKeeper allocates space in the transaction log file in +# blocks of preAllocSize kilobytes. The default block size is 64M. One reason +# for changing the size of the blocks is to reduce the block size if snapshots +# are taken more often. (Also, see snapCount). +#preAllocSize=65536 + +# Clients can submit requests faster than ZooKeeper can process them, +# especially if there are a lot of clients. To prevent ZooKeeper from running +# out of memory due to queued requests, ZooKeeper will throttle clients so that +# there is no more than globalOutstandingLimit outstanding requests in the +# system. The default limit is 1,000.ZooKeeper logs transactions to a +# transaction log. After snapCount transactions are written to a log file a +# snapshot is started and a new transaction log file is started. The default +# snapCount is 10,000. +#snapCount=1000 + +# If this option is defined, requests will be will logged to a trace file named +# traceFile.year.month.day. +#traceFile= + +# Leader accepts client connections. Default value is "yes". The leader machine +# coordinates updates. For higher update throughput at thes slight expense of +# read throughput the leader can be configured to not accept clients and focus +# on coordination. +#leaderServes=yes + +# Autopurge every hour to avoid using lots of disk in bursts +# Order of the next 2 properties matters. +# autopurge.snapRetainCount must be before autopurge.purgeInterval. +autopurge.snapRetainCount=3 +autopurge.purgeInterval=1 \ No newline at end of file diff --git a/lib/zookeeper b/lib/zookeeper new file mode 100644 index 0000000000..e62ba8ada8 --- /dev/null +++ b/lib/zookeeper @@ -0,0 +1,86 @@ +#!/bin/bash +# +# lib/zookeeper +# Functions to control the installation and configuration of **zookeeper** + +# Dependencies: +# +# - ``functions`` file + +# ``stack.sh`` calls the entry points in this order: +# +# - is_zookeeper_enabled +# - install_zookeeper +# - configure_zookeeper +# - init_zookeeper +# - start_zookeeper +# - stop_zookeeper +# - cleanup_zookeeper + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set +o xtrace + + +# Defaults +# -------- + +# + +# Set up default directories +ZOOKEEPER_DATA_DIR=$DEST/data/zookeeper +ZOOKEEPER_CONF_DIR=/etc/zookeeper + + +# Entry Points +# ------------ + +# Test if any zookeeper service us enabled +# is_zookeeper_enabled +function is_zookeeper_enabled { + [[ ,${ENABLED_SERVICES}, =~ ,"zookeeper", ]] && return 0 + return 1 +} + +# cleanup_zookeeper() - Remove residual data files, anything left over from previous +# runs that a clean run would need to clean up +function cleanup_zookeeper { + sudo rm -rf $ZOOKEEPER_DATA_DIR +} + +# configure_zookeeper() - Set config files, create data dirs, etc +function configure_zookeeper { + sudo cp $FILES/zookeeper/* $ZOOKEEPER_CONF_DIR + sudo sed -i -e 's|.*dataDir.*|dataDir='$ZOOKEEPER_DATA_DIR'|' $ZOOKEEPER_CONF_DIR/zoo.cfg +} + +# init_zookeeper() - Initialize databases, etc. +function init_zookeeper { + # clean up from previous (possibly aborted) runs + # create required data files + sudo rm -rf $ZOOKEEPER_DATA_DIR + sudo mkdir -p $ZOOKEEPER_DATA_DIR +} + +# install_zookeeper() - Collect source and prepare +function install_zookeeper { + install_package zookeeperd +} + +# start_zookeeper() - Start running processes, including screen +function start_zookeeper { + start_service zookeeper +} + +# stop_zookeeper() - Stop running processes (non-screen) +function stop_zookeeper { + stop_service zookeeper +} + +# Restore xtrace +$XTRACE + +# Tell emacs to use shell-script-mode +## Local variables: +## mode: shell-script +## End: diff --git a/stack.sh b/stack.sh index bdbb025a0b..118785150a 100755 --- a/stack.sh +++ b/stack.sh @@ -539,6 +539,7 @@ source $TOP_DIR/lib/heat source $TOP_DIR/lib/neutron-legacy source $TOP_DIR/lib/ldap source $TOP_DIR/lib/dstat +source $TOP_DIR/lib/zookeeper # Extras Source # -------------- @@ -729,6 +730,11 @@ run_phase stack pre-install install_rpc_backend +if is_service_enabled zookeeper; then + cleanup_zookeeper + configure_zookeeper + init_zookeeper +fi if is_service_enabled $DATABASE_BACKENDS; then install_database fi @@ -968,6 +974,15 @@ save_stackenv $LINENO start_dstat +# Zookeeper +# ----- + +# zookeeper for use with tooz for Distributed Lock Management capabilities etc., +if is_service_enabled zookeeper; then + start_zookeeper +fi + + # Keystone # -------- diff --git a/stackrc b/stackrc index 819aa0125c..19538c0679 100644 --- a/stackrc +++ b/stackrc @@ -69,7 +69,7 @@ if ! isset ENABLED_SERVICES ; then # Dashboard ENABLED_SERVICES+=,horizon # Additional services - ENABLED_SERVICES+=,rabbit,tempest,mysql,dstat + ENABLED_SERVICES+=,rabbit,tempest,mysql,dstat,zookeeper fi # SQLAlchemy supports multiple database drivers for each database server diff --git a/unstack.sh b/unstack.sh index 30447a7005..0cace3254a 100755 --- a/unstack.sh +++ b/unstack.sh @@ -69,6 +69,7 @@ source $TOP_DIR/lib/heat source $TOP_DIR/lib/neutron-legacy source $TOP_DIR/lib/ldap source $TOP_DIR/lib/dstat +source $TOP_DIR/lib/zookeeper # Extras Source # -------------- @@ -172,6 +173,10 @@ if is_service_enabled dstat; then stop_dstat fi +if is_service_enabled zookeeper; then + stop_zookeeper +fi + # Clean up the remainder of the screen processes SCREEN=$(which screen) if [[ -n "$SCREEN" ]]; then