From b297146ea8423f9a513e1365c134d6d6611ced5a Mon Sep 17 00:00:00 2001 From: Denis Makogon Date: Tue, 13 Dec 2016 02:26:36 +0200 Subject: [PATCH] Issue #20: Keep devstack plugin in-tree Implements #20 --- devstack/README.rst | 59 +++++++ devstack/files/debs/functions | 2 + devstack/files/rpms-suse/functions | 3 + devstack/files/rpms/functions | 3 + devstack/plugin.sh | 250 +++++++++++++++++++++++++++++ devstack/settings | 30 ++++ 6 files changed, 347 insertions(+) create mode 100644 devstack/README.rst create mode 100644 devstack/files/debs/functions create mode 100644 devstack/files/rpms-suse/functions create mode 100644 devstack/files/rpms/functions create mode 100644 devstack/plugin.sh create mode 100644 devstack/settings diff --git a/devstack/README.rst b/devstack/README.rst new file mode 100644 index 0000000..b0e7372 --- /dev/null +++ b/devstack/README.rst @@ -0,0 +1,59 @@ +Enabling Picasso (Functions-as-a-Service) in DevStack +===================================================== + +Installing Glide +================ + +Note that your machine should have Glide installed. +See more info at https://github.com/Masterminds/glide + + +Download DevStack +================= + +.. sourcecode:: bash + + export DEVSTACK_DIR=~/devstack + git clone git://git.openstack.org/openstack-dev/devstack.git $DEVSTACK_DIR + +Enable the FaaS plugin +====================== + +Enable the plugin by adding the following section to ``$DEVSTACK_DIR/local.conf`` + +.. sourcecode:: bash + + [[local|localrc]] + + enable_plugin picasso git@github.com:iron-io/picasso.git + + # Picasso configuration + PICASSO_REPO=${PICASSO_REPO:-git@github.com:iron-io/picasso.git} + PICASSO_BRANCH=${PICASSO_BRANCH:-master} + PICASSO_DIR=${PICASSO_DIR:-${DEST}/picasso} + PICASSO_PORT=${PICASSO_PORT:-10001} + PICASSO_LOG_LEVEL=${PICASSO_LOG_LEVEL:-DEBUG} + PICASSO_LOG_FILE=${PICASSO_LOG_FILE:-/var/log/picasso-api.log} + + # Picasso client configuration + PICASSO_CLIENT_REPO=${PICASSO_CLIENT_REPO:-git@github.com:iron-io/python-picassoclient.git} + PICASSO_CLIENT_DIR=${PICASSO_CLIENT_DIR:-${DEST}/python-picassoclient} + PICASSO_CLIENT_BRANCH=${PICASSO_CLIENT_BRANCH:-master} + + # IronFunctions parameters + FUNCTIONS_REPO=${FUNCTIONS_REPO:-git@github.com:iron-io/functions.git} + FUNCTIONS_BRANCH=${FUNCTIONS_BRANCH:-master} + FUNCTIONS_PORT=${FUNCTIONS_PORT:-10501} + FUNCTIONS_DB=${FUNCTIONS_DBPATH:-bolt://$FUNCTIONS_DIR/devstack.functions.storage.db?bucket=funcs} + FUNCTIONS_MQ=${FUNCTIONS_DBPATH:-bolt://$FUNCTIONS_DIR/devstack.functions.queue.db} + FUNCTIONS_LOG_LEVEL=${FUNCTIONS_LOG_LEVEL:-DEBUG} + + DOCKERD_OPTS=${DOCKERD_OPTS:---dns 8.8.8.8 --dns 8.8.4.4 --storage-driver=overlay2 -H fd://} + +Run the DevStack utility +======================== + +.. sourcecode:: bash + + cd $DEVSTACK_DIR + ./stack.sh diff --git a/devstack/files/debs/functions b/devstack/files/debs/functions new file mode 100644 index 0000000..30ba113 --- /dev/null +++ b/devstack/files/debs/functions @@ -0,0 +1,2 @@ +golang +docker \ No newline at end of file diff --git a/devstack/files/rpms-suse/functions b/devstack/files/rpms-suse/functions new file mode 100644 index 0000000..dbd341c --- /dev/null +++ b/devstack/files/rpms-suse/functions @@ -0,0 +1,3 @@ +golang +hg +docker \ No newline at end of file diff --git a/devstack/files/rpms/functions b/devstack/files/rpms/functions new file mode 100644 index 0000000..dbd341c --- /dev/null +++ b/devstack/files/rpms/functions @@ -0,0 +1,3 @@ +golang +hg +docker \ No newline at end of file diff --git a/devstack/plugin.sh b/devstack/plugin.sh new file mode 100644 index 0000000..b7592e7 --- /dev/null +++ b/devstack/plugin.sh @@ -0,0 +1,250 @@ +#!/bin/bash +# + +# lib/functions +# Functions to control the configuration and operation of the **Functions-as-a-Service** service + +# Dependencies: +# ``functions`` file +# ``DEST``, ``STACK_USER`` must be defined +# ``SERVICE_{HOST|PROTOCOL|TOKEN}`` must be defined + +# ``stack.sh`` calls the entry points in this order: +# + + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set +o xtrace + +# Functions +# --------- + + +function is_functions_enabled { + [[ ,${ENABLED_SERVICES} =~ ,"picasso" ]] && return 0 + return 1 +} + +function create_functions_accounts { + if [[ "$ENABLED_SERVICES" =~ "functions" ]]; then + + create_service_user "picasso" "admin" + + local functions_service=$(get_or_create_service "picasso" \ + "functions" "Functions-as-a-Service") + get_or_create_endpoint $functions_service \ + "$REGION_NAME" \ + "http://$SERVICE_HOST:$PICASSO_PORT" \ + "http://$SERVICE_HOST:$PICASSO_PORT" \ + "http://$SERVICE_HOST:$PICASSO_PORT" + fi +} + +function install_go_1.7.1 { + wget https://storage.googleapis.com/golang/go1.7.linux-amd64.tar.gz -O /tmp/go1.7.linux-amd64.tar.gz + sudo tar -xvf /tmp/go1.7.linux-amd64.tar.gz -C /usr/local/ + sudo rm -fr /usr/bin/go + sudo rm -fr /usr/bin/gofmt + sudo ln -s /usr/local/go/bin/go /usr/bin/go + sudo ln -s /usr/local/go/bin/gofmt /usr/bin/gofmt + `which go` env +} + +function check_docker { + if is_ubuntu; then + dpkg -s docker-engine > /dev/null 2>&1 + else + rpm -q docker-engine > /dev/null 2>&1 + fi +} + +function install_docker { + check_docker || curl -fsSL https://get.docker.com/ | sudo sh + + echo "Adding ${STACK_USER} to ${docker_group}..." + add_user_to_group $STACK_USER $DOCKER_GROUP + echo "Adding $(whoami) to ${DOCKER_GROUP}..." + add_user_to_group $(whoami) $DOCKER_GROUP + + if is_fedora; then + install_package socat dnsmasq + fi + + if is_ubuntu && [ $UBUNTU_RELEASE_BASE_NUM -le 14 ]; then + sudo service docker start || true + else + echo -e "[Unit] + Description=Docker Application Container Engine + Documentation=https://docs.docker.com + After=network.target docker.socket + Requires=docker.socket + [Service] + Type=notify + Environment=SERVICE_HOST=${SERVICE_HOST} + ExecStart=/usr/bin/dockerd -H tcp://${SERVICE_HOST}:2375 ${DOCKERD_OPTS} + MountFlags=slave + LimitNOFILE=1048576 + LimitNPROC=1048576 + LimitCORE=infinity + TimeoutStartSec=0 + Delegate=yes + [Install] + WantedBy=multi-user.target" | sudo tee -a /lib/systemd/system/docker.service.new + + sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service + + sudo systemctl daemon-reload + sudo systemctl enable docker.service + sudo systemctl restart docker || true + fi +} + + +function is_glide_installed { + echo_summary "Attempting to find Glide binary" + local glbin=`which glide` + if [[ -z "${glbin}" ]]; then + echo_summary "Glide is not installed, aborting." + exit 1 + fi +} + + +function install_picasso { + echo_summary "Installing Picasso" + git_clone $PICASSO_REPO $PICASSO_DIR $FUNCTIONS_BRANCH + pushd $PICASSO_DIR && docker -H tcp://${SERVICE_HOST}:2375 build -t picasso-api -f $PICASSO_DIR/Dockerfile .; popd +} + +function apply_picasso_migrations { + echo_summary "Applying Picasso migrations" + recreate_database_mysql functions + local picasso_db=$(get_database_type_mysql)://${DATABASE_USER}:${DATABASE_PASSWORD}@${MYSQL_HOST}/functions + pushd $PICASSO_DIR && PICASSO_MIGRATIONS_DB=${picasso_db} alembic upgrade head; popd + +} + +function install_functions { + echo_summary "Pulling Functions sources" + git_clone $FUNCTIONS_REPO $GOPATH/src/github.com/iron-io/functions $FUNCTIONS_BRANCH + pushd $FUNCTIONS_DIR && GOPATH=${GOPATH} make all; popd +} + +function configure_systemctl_for_functions_api { + echo -e "[Unit] +Description=Functions API service +After=network.target +[Service] +Type=notify +Environment=GOPATH=${GOPATH} +Environment=PORT=${FUNCTIONS_PORT} +Environment=API_URL=http://${SERVICE_HOST}:${FUNCTIONS_PORT} +Environment=DOCKER_HOST=tcp://${SERVICE_HOST}:2375 +Environment=LOG_LEVEL=${FUNCTIONS_LOG_LEVEL} +Environment=DB_URL=${FUNCTIONS_DB} +Environment=MQ_URL=${FUNCTIONS_MQ} +ExecStart=${FUNCTIONS_DIR}/functions +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=0 +[Install] +WantedBy=multi-user.target" | sudo tee - a /lib/systemd/system/functions-api.service.new + + sudo mv /lib/systemd/system/functions-api.service.new /lib/systemd/system/functions-api.service + sudo systemctl daemon-reload +} + +function configure_systemctl_for_picasso_api { + rm -fr $PICASSO_DIR/.picasso.docker + echo -e "PICASSO_HOST=0.0.0.0 +PICASSO_PORT=${PICASSO_PORT} +PICASSO_DB=$(get_database_type_mysql)://${DATABASE_USER}:${DATABASE_PASSWORD}@${SERVICE_HOST}/functions +KEYSTONE_ENDPOINT=http://${SERVICE_HOST}:5000/v3 +FUNCTIONS_URL=http://${SERVICE_HOST}:${FUNCTIONS_PORT}/v1 +PICASSO_LOG_LEVEL=DEBUG + " | tee -a $PICASSO_DIR/.picasso.docker + + echo -e "[Unit] +Description=Picasso API service +After=network.target +[Service] +Type=notify +Environment=PICASSO_DIR=${PICASSO_DIR} +Environment=DOCKER_HOST=tcp://${SERVICE_HOST}:2375 +ExecStart=/usr/bin/docker -H tcp://${SERVICE_HOST}:2375 run --name picasso-api -d -p ${SERVICE_HOST}:10001:10001 --env-file ${PICASSO_DIR}/.picasso.docker picasso-api +ExecStop=/usr/bin/docker -H tcp://${SERVICE_HOST}:2375 stop -t 1 picasso-api +ExecStopPost=/usr/bin/docker -H tcp://${SERVICE_HOST}:2375 rm -f picasso-api +RemainAfterExit=true +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=0 +[Install] +WantedBy=multi-user.target" | sudo tee - a /lib/systemd/system/picasso-api.service.new + + sudo mv /lib/systemd/system/picasso-api.service.new /lib/systemd/system/picasso-api.service + sudo systemctl daemon-reload +} + + +function start_picasso_api { + echo_summary "Starting Picasso API" + sudo systemctl start picasso-api.service +} + + +function stop_picasso_api { + echo_summary "Stopping Picasso API" + sudo systemctl stop picasso-api.service +} + + +function start_functions { + echo_summary "Starting Functions API" + sudo systemctl start functions-api.service & +} + + +function stop_functions { + echo_summary "Stopping Functions API" + sudo systemctl stop functions-api.service & +} + +function install_python_picassoclient { + git_clone $PICASSO_CLIENT_REPO $PICASSO_CLIENT_DIR $PICASSO_CLIENT_BRANCH + setup_develop $PICASSO_CLIENT_DIR +} + +if is_service_enabled functions; then + + if [[ "$1" == "stack" && "$2" == "install" ]]; then + echo_summary "Installing Functions" + install_docker + install_go_1.7.1 + is_glide_installed + install_functions + install_picasso + install_python_picassoclient + elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then + echo_summary "Configuring Picasso and IronFunctions" + configure_systemctl_for_functions_api + configure_systemctl_for_picasso_api + if is_service_enabled key; then + create_functions_accounts + fi + elif [[ "$1" == "stack" && "$2" == "extra" ]]; then + apply_picasso_migrations + start_functions + start_picasso_api + fi + + if [[ "$1" == "unstack" ]]; then + stop_functions + stop_picasso_api + fi +fi + +# Restore xtrace +$XTRACE diff --git a/devstack/settings b/devstack/settings new file mode 100644 index 0000000..2f63571 --- /dev/null +++ b/devstack/settings @@ -0,0 +1,30 @@ +# Settings needed for Picasso (Functions-as-a-Service) plugin +# ----------------------------------------------------------- + +# Set up default directories + +# Picasso service configuration +PICASSO_REPO=${PICASSO_REPO:-git@github.com:iron-io/picasso.git} +PICASSO_BRANCH=${PICASSO_BRANCH:-master} +PICASSO_DIR=${PICASSO_DIR:-${DEST}/picasso} +PICASSO_PORT=${PICASSO_PORT:-10001} +PICASSO_LOG_LEVEL=${PICASSO_LOG_LEVEL:-DEBUG} + +# Picasso client configuration +PICASSO_CLIENT_REPO=${PICASSO_CLIENT_REPO:-git@github.com:iron-io/python-picassoclient.git} +PICASSO_CLIENT_DIR=${PICASSO_CLIENT_DIR:-${DEST}/python-picassoclient} +PICASSO_CLIENT_BRANCH=${PICASSO_CLIENT_BRANCH:-master} + +# IronFunctions configuration +GOPATH=${GOPATH:-${DEST}/functions} +FUNCTIONS_DIR=${FUNCTIONS_DIR:-${GOPATH}/src/github.com/iron-io/functions} +FUNCTIONS_REPO=${FUNCTIONS_REPO:-git@github.com:iron-io/functions.git} +FUNCTIONS_BRANCH=${FUNCTIONS_BRANCH:-master} +FUNCTIONS_PORT=${FUNCTIONS_PORT:-10501} +FUNCTIONS_DB=${FUNCTIONS_DBPATH:-bolt://$FUNCTIONS_DIR/devstack.functions.storage.db?bucket=funcs} +FUNCTIONS_MQ=${FUNCTIONS_DBPATH:-bolt://$FUNCTIONS_DIR/devstack.functions.queue.db} +FUNCTIONS_LOG_LEVEL=${FUNCTIONS_LOG_LEVEL:-DEBUG} + +DOCKERD_OPTS=${DOCKER_OPTS:---dns 8.8.8.8 --dns 8.8.4.4 --storage-driver=overlay2} + +enable_service picasso