Enhanced refresh monasca transform script
refresh_monasca_script.sh is useful for development in a devstack environment. Enhanced the script to * start and stop monasca-transform process running in a screen session * added more hardening to catch for errors and exit if any command fails. * added debugging statements help track down any errors when the script is run. Change-Id: Idab02d555eed192d8242c870017955b935532c3d
This commit is contained in:
Ashwin Agate
parent
699103e345
commit
5c15a99a5c
@ -87,7 +87,10 @@ Once the deploy is up use the following commands to set up tox.
|
||||
|
||||
To regenerate the environment for development purposes a script is provided
|
||||
on the devstack instance at
|
||||
/opt/stack/monasca-transform/tools/vagrant/refresh_monasca_transform.sh
|
||||
To run the refresh_monasca_transform.sh script on devstack instance
|
||||
|
||||
cd /opt/stack/monasca-transform
|
||||
tools/vagrant/refresh_monasca_transform.sh
|
||||
|
||||
(note: to use/run tox after running this script, the
|
||||
@ -95,10 +98,18 @@ on the devstack instance at
|
||||
|
||||
This mostly re-does the work of the devstack plugin, updating the code from the
|
||||
shared directory, regenerating the venv and the zip that is passed to spark
|
||||
during the spark-submit call. The configuration and the contents of the
|
||||
database are updated with fresh copies also though the start scripts, driver and
|
||||
service python code are left as they are (because I'm not envisaging much change
|
||||
in those).
|
||||
during the spark-submit call. The configuration and the transform and
|
||||
pre transform specs in the database are updated with fresh copies, along
|
||||
with driver and service python code.
|
||||
|
||||
If refresh_monasca_transform.sh script completes successfully you should see
|
||||
a message like the following in the console.
|
||||
|
||||
***********************************************
|
||||
* *
|
||||
* SUCCESS!! refresh monasca transform done. *
|
||||
* *
|
||||
***********************************************
|
||||
|
||||
### Development workflow
|
||||
|
||||
@ -157,6 +168,19 @@ For example:
|
||||
|
||||
Reference: https://wiki.openstack.org/wiki/Testr
|
||||
|
||||
## Access Spark Streaming and Spark Master/Worker User Interface
|
||||
|
||||
In a devstack environment ports on which Spark Streaming UI (4040), Spark Master(18080)
|
||||
and Spark Worker (18081) UI are available are forwarded to the host and are
|
||||
accessible from the host machine.
|
||||
|
||||
http://<host_machine_ip>:4040/ (Note: Spark Streaming UI,
|
||||
is available only when
|
||||
monasca-transform application
|
||||
is running)
|
||||
http://<host_machine_ip>:18080/ (Spark Master UI)
|
||||
http://<host_machine_ip>:18081/ (Spark Worker UI)
|
||||
|
||||
## To run monasca-transform using a different deployment technology
|
||||
|
||||
Monasca-transform requires supporting services, such as Kafka and
|
||||
|
||||
@ -73,6 +73,9 @@ class Transform(os_service.Service):
|
||||
'Shutting down all threads and exiting')
|
||||
shutdown_all_threads_and_die()
|
||||
|
||||
def stop(self, graceful):
|
||||
shutdown_all_threads_and_die()
|
||||
|
||||
|
||||
class TransformService(threading.Thread):
|
||||
|
||||
|
||||
@ -3,6 +3,13 @@
|
||||
SCRIPT_HOME=$(dirname $(readlink -f $BASH_SOURCE))
|
||||
pushd $SCRIPT_HOME
|
||||
|
||||
echo "create_zip.py: creating a zip file at ../monasca_transform/monasca-transform.zip..."
|
||||
python create_zip.py
|
||||
|
||||
popd
|
||||
rc=$?
|
||||
if [[ $rc == 0 ]]; then
|
||||
echo "created zip file at ../monasca_transfom/monasca-transform.zip sucessfully"
|
||||
else
|
||||
echo "error creating zip file at ../monasca_transform/monasca-transform.zip, bailing out"
|
||||
exit 1
|
||||
fi
|
||||
popd
|
||||
|
||||
@ -3,7 +3,29 @@
|
||||
SCRIPT_HOME=$(dirname $(readlink -f $BASH_SOURCE))
|
||||
pushd $SCRIPT_HOME
|
||||
|
||||
python ddl/generate_ddl.py -t pre_transform_spec -i ddl/pre_transform_specs_template.sql -s ../monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json -o ddl/pre_transform_specs.sql
|
||||
python ddl/generate_ddl.py -t transform_spec -i ddl/transform_specs_template.sql -s ../monasca_transform/data_driven_specs/transform_specs/transform_specs.json -o ddl/transform_specs.sql
|
||||
PRE_TRANSFORM_SPECS_JSON="../monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json"
|
||||
PRE_TRANSFORM_SPECS_SQL="ddl/pre_transform_specs.sql"
|
||||
|
||||
popd
|
||||
TRANSFORM_SPECS_JSON="../monasca_transform/data_driven_specs/transform_specs/transform_specs.json"
|
||||
TRANSFORM_SPECS_SQL="ddl/transform_specs.sql"
|
||||
|
||||
echo "converting {$PRE_TRANSFORM_SPECS_JSON} to {$PRE_TRANSFORM_SPECS_SQL} ..."
|
||||
python ddl/generate_ddl.py -t pre_transform_spec -i ddl/pre_transform_specs_template.sql -s "$PRE_TRANSFORM_SPECS_JSON" -o "$PRE_TRANSFORM_SPECS_SQL"
|
||||
rc=$?
|
||||
if [[ $rc == 0 ]]; then
|
||||
echo "converting {$PRE_TRANSFORM_SPECS_JSON} to {$PRE_TRANSFORM_SPECS_SQL} sucessfully..."
|
||||
else
|
||||
echo "error in converting {$PRE_TRANSFORM_SPECS_JSON} to {$PRE_TRANSFORM_SPECS_SQL}, bailing out"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "converting {$TRANSFORM_SPECS_JSON} to {$TRANSFORM_SPECS_SQL}..."
|
||||
python ddl/generate_ddl.py -t transform_spec -i ddl/transform_specs_template.sql -s "$TRANSFORM_SPECS_JSON" -o "$TRANSFORM_SPECS_SQL"
|
||||
rc=$?
|
||||
if [[ $rc == 0 ]]; then
|
||||
echo "converting {$TRANSFORM_SPECS_JSON} to {$TRANSFORM_SPECS_SQL} sucessfully..."
|
||||
else
|
||||
echo "error in converting {$TRANSFORM_SPECS_JSON} to {$TRANSFORM_SPECS_SQL}, bailing out"
|
||||
exit 1
|
||||
fi
|
||||
popd
|
||||
|
||||
@ -1,40 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
#
|
||||
# for debugging
|
||||
#
|
||||
|
||||
## turn on display command
|
||||
## set -x
|
||||
|
||||
if grep -q devstack <<<`hostname`; then
|
||||
echo Refreshing monasca-transform
|
||||
else
|
||||
echo Yikes, no - this is not devstack!
|
||||
exit 1
|
||||
fi
|
||||
if [ -d "/home/vagrant/devstack" ] ; then
|
||||
|
||||
. /home/vagrant/devstack/.stackenv
|
||||
if [ -d ~/devstack ] ; then
|
||||
|
||||
. ~/devstack/.stackenv
|
||||
|
||||
fi
|
||||
|
||||
SCRIPT_HOME=$(dirname $(readlink -f $BASH_SOURCE))
|
||||
pushd $SCRIPT_HOME
|
||||
|
||||
# TODO not sure how to stop monasca-transform from here now that
|
||||
# the control for it in DevStack is via screen -x stack
|
||||
if grep -q running <<<`sudo service monasca-transform status`; then
|
||||
sudo service monasca-transform stop
|
||||
else
|
||||
echo "monasca-transform service not running"
|
||||
# stop monasca-transform process running in screen session
|
||||
STOP_SLEEP=10
|
||||
if [[ -r /opt/stack/status/stack/monasca-transform.pid ]]; then
|
||||
echo "going to shutdown $service running in screen session..."
|
||||
pkill -g $(cat /opt/stack/status/stack/monasca-transform.pid)
|
||||
rc=$?
|
||||
if [[ $rc == 0 ]]; then
|
||||
echo "waiting $STOP_SLEEP seconds for monasca-transform to exit..."
|
||||
sleep $STOP_SLEEP
|
||||
screen -S stack -p monasca-transform -X stuff "\015"
|
||||
echo "monasca-transform process stopped sucessfully"
|
||||
else
|
||||
echo "monasca-transform process wasnt running, proceeding"
|
||||
fi
|
||||
fi
|
||||
|
||||
#
|
||||
# turn on exit immediately if command fails
|
||||
#
|
||||
set -e
|
||||
|
||||
sudo rm -rf /home/vagrant/monasca-transform-source /home/vagrant/monasca-transform
|
||||
|
||||
echo "calling setup_local_repos.sh ..."
|
||||
sudo ./setup_local_repos.sh
|
||||
rc=$?
|
||||
if [[ $rc == 0 ]]; then
|
||||
echo "setup_local_repos.sh completed sucessfully..."
|
||||
else
|
||||
echo "Error in setup_local_repos.sh, bailing out"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# update the database with configuration
|
||||
echo "populating monasca-transform db tables with pre_transform_specs and transform_specs..."
|
||||
sudo cp /home/vagrant/monasca-transform/scripts/ddl/pre_transform_specs.sql /opt/monasca/transform/lib/pre_transform_specs.sql
|
||||
sudo cp /home/vagrant/monasca-transform/scripts/ddl/transform_specs.sql /opt/monasca/transform/lib/transform_specs.sql
|
||||
sudo mysql -h "127.0.0.1" -um-transform -ppassword < /opt/monasca/transform/lib/pre_transform_specs.sql
|
||||
sudo mysql -h "127.0.0.1" -um-transform -ppassword < /opt/monasca/transform/lib/transform_specs.sql
|
||||
echo "populating monasca-transform db tables with pre_transform_specs and transform_specs done."
|
||||
|
||||
# update the zip file used for spark submit
|
||||
echo "copying new monasca-transform.zip to /opt/monasca/transform/lib/ ..."
|
||||
sudo cp /home/vagrant/monasca-transform/scripts/monasca-transform.zip /opt/monasca/transform/lib/.
|
||||
echo "copying new monasca-transform.zip to /opt/monasca/transform/lib/ done."
|
||||
|
||||
# update the configuration file
|
||||
sudo cp /home/vagrant/monasca-transform/devstack/files/monasca-transform/monasca-transform.conf /etc/.
|
||||
@ -42,18 +75,31 @@ if [ -n "$SERVICE_HOST" ]; then
|
||||
sudo sudo sed -i "s/brokers=192\.168\.15\.6:9092/brokers=${SERVICE_HOST}:9092/g" /etc/monasca-transform.conf
|
||||
fi
|
||||
|
||||
# delete the venv
|
||||
sudo rm -rf /opt/monasca/transform/venv
|
||||
|
||||
# refresh the monasca-transform code to /opt/stack
|
||||
sudo rm -rf /opt/stack/monasca-transform
|
||||
MONASCA_TRANSFORM_VENV="/opt/monasca/transform/venv"
|
||||
MONASCA_TRANSFORM_DIR="/opt/stack/monasca-transform"
|
||||
|
||||
echo "refreshing $MONASCA_TRANSFORM_DIR..."
|
||||
|
||||
# turn on display command
|
||||
set -x
|
||||
|
||||
# delete the venv
|
||||
sudo rm -rf $MONASCA_TRANSFORM_VENV
|
||||
# refresh the monasca-transform code at /opt/stack
|
||||
sudo rm -rf "$MONASCA_TRANSFORM_DIR"
|
||||
pushd /opt/stack
|
||||
sudo git clone /home/vagrant/monasca-transform
|
||||
sudo chown -R vagrant:vagrant /opt/stack/monasca-transform
|
||||
virtualenv /opt/monasca/transform/venv
|
||||
. /opt/monasca/transform/venv/bin/activate
|
||||
pip install -e /opt/stack/monasca-transform/
|
||||
sudo chown -R vagrant:vagrant "$MONASCA_TRANSFORM_DIR"
|
||||
virtualenv "$MONASCA_TRANSFORM_VENV"
|
||||
. "$MONASCA_TRANSFORM_VENV"/bin/activate
|
||||
pip install -e "$MONASCA_TRANSFORM_DIR"
|
||||
deactivate
|
||||
|
||||
# turn off display command
|
||||
set +x
|
||||
|
||||
echo "refreshing $MONASCA_TRANSFORM_DIR done."
|
||||
popd
|
||||
|
||||
function get_id () {
|
||||
@ -62,10 +108,26 @@ function get_id () {
|
||||
|
||||
source ~/devstack/openrc admin admin
|
||||
export ADMIN_PROJECT_ID=$(get_id openstack project show admin)
|
||||
echo "updating publish_kafka_project_id to $ADMIN_PROJECT_ID in /etc/monasca-transform.conf..."
|
||||
sudo sed -i "s/publish_kafka_project_id=d2cb21079930415a9f2a33588b9f2bb6/publish_kafka_project_id=${ADMIN_PROJECT_ID}/g" /etc/monasca-transform.conf
|
||||
echo "updating publish_kafka_project_id to $ADMIN_PROJECT_ID in /etc/monasca-transform.conf done."
|
||||
|
||||
# TODO not sure how to start monasca-transform from here now that
|
||||
# the control for it in DevStack is via screen -x stack
|
||||
sudo service monasca-transform start
|
||||
# start monasca-transform in screen session
|
||||
start_command="/etc/monasca/transform/init/start-monasca-transform.sh"
|
||||
screen -S stack -p monasca-transform -X stuff "$start_command & echo \$! >/opt/stack/status/stack/monasca-transform.pid; fg || echo \"monasca-transform failed to start\""
|
||||
screen -S stack -p monasca-transform -X stuff "\015"
|
||||
rc=$?
|
||||
if [[ $rc == 0 ]]; then
|
||||
echo "monasca-transform process started sucessfully"
|
||||
else
|
||||
echo "Error: monasca-transfrom process was not started. Please check screen session for error messages"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "***********************************************"
|
||||
echo "* *"
|
||||
echo "* SUCCESS!! refresh monasca transform done. *"
|
||||
echo "* *"
|
||||
echo "***********************************************"
|
||||
|
||||
popd
|
||||
|
||||
@ -1,24 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
echo "going to copy /monasca-transform-source to /home/vagrant ..."
|
||||
rsync -a --exclude='tools/vagrant/.vagrant' /monasca-transform-source /home/vagrant/
|
||||
|
||||
echo "going to move /home/vagrant/monasca-transform-source /home/vagrant/monasca-transform..."
|
||||
mv /home/vagrant/monasca-transform-source /home/vagrant/monasca-transform
|
||||
pushd /home/vagrant/monasca-transform
|
||||
|
||||
# prepare the codebase
|
||||
#
|
||||
# generate the sql scripts to populate the database
|
||||
# generate the sql scripts which will populate the database
|
||||
# with the pre_transform and transform specs
|
||||
scripts/generate_ddl.sh
|
||||
# build the zip
|
||||
scripts/create_zip.sh
|
||||
rc=$?
|
||||
if [[ $rc == 0 ]]; then
|
||||
echo "scripts/generate_ddl.sh completed sucessfully..."
|
||||
else
|
||||
echo "Error in scripts/generate_ddl.sh, bailing out"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# build the zip (to be submitted with spark application)
|
||||
scripts/create_zip.sh
|
||||
rc=$?
|
||||
if [[ $rc == 0 ]]; then
|
||||
echo "scripts/create_zip.sh completed sucessfully..."
|
||||
else
|
||||
echo "Error in scripts/create_zip.sh, bailing out"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "creating a local commit..."
|
||||
git config --global user.email "local.devstack.committer@hpe.com"
|
||||
git config --global user.name "Local devstack committer"
|
||||
git add --all
|
||||
git commit -m "Local commit"
|
||||
echo "creating a local commit done."
|
||||
|
||||
CURRENT_BRANCH=`git status | grep 'On branch' | sed 's/On branch //'`
|
||||
if [ ${CURRENT_BRANCH} != 'master' ]
|
||||
then
|
||||
echo Maintaining current branch ${CURRENT_BRANCH}
|
||||
echo "Maintaining current branch ${CURRENT_BRANCH}"
|
||||
# set the branch to what we're using in local.conf
|
||||
if [[ -z `grep ${CURRENT_BRANCH} /home/vagrant/devstack/local.conf` ]]; then
|
||||
sed -i "s/enable_plugin monasca-transform \/home\/vagrant\/monasca-transform//g" /home/vagrant/devstack/local.conf
|
||||
@ -27,5 +50,4 @@ then
|
||||
printf "# END DEVSTACK LOCAL.CONF CONTENTS" >> /home/vagrant/devstack/local.conf
|
||||
fi
|
||||
fi
|
||||
|
||||
popd
|
||||
|
||||
Loading…
Reference in New Issue
Block a user