Add cinder support

* using the new functional pattern: cinder_XXX functions are in
  lib/cinder
* enable with 'c-api,c-sch,c-vol' in ENABLED_SERVICES, n-vol is still the default
* exercises/volumes.sh runs for cinder and n-vol
* move config to /etc/cinder
* change volume_group to stack-volumes; this also renames the backing file
  to /opt/stack/data/stack-volumes-backing-file.
* removes osapi_volume from nova.conf enabled_apis
* integrates cinder + keystone
* launches c-sch
* tweaks for multi node
* move enabled_apis substitution to init_cinder

18Jun2010
* restored & rebased
* update setup.py to use setup_develop() in lib/cinder

Change-Id: I1e1aa4387031c56e4fa239eb73bea2af8cef0e38

exerciserc

@@ -20,3 +20,9 @@ export RUNNING_TIMEOUT=${RUNNING_TIMEOUT:-$(($BOOT_TIMEOUT + $ACTIVE_TIMEOUT))}
 
 # Max time to wait for a vm to terminate
 export TERMINATE_TIMEOUT=${TERMINATE_TIMEOUT:-30}
+
+# Max time to wait for a euca-volume command to propogate
+export VOLUME_TIMEOUT=${VOLUME_TIMEOUT:-30}
+
+# Max time to wait for a euca-delete command to propogate
+export VOLUME_DELETE_TIMEOUT=${SNAPSHOT_DELETE_TIMEOUT:-60}

exercises/euca.sh

@@ -24,6 +24,9 @@ set -o xtrace
 # Keep track of the current directory
 EXERCISE_DIR=$(cd $(dirname "$0") && pwd)
 TOP_DIR=$(cd $EXERCISE_DIR/..; pwd)
+VOLUME_ZONE=cinder
+VOLUME_SIZE=1
+ATTACH_DEVICE=/dev/vdc
 
 # Import common functions
 source $TOP_DIR/functions

exercises/volumes.sh

@@ -33,6 +33,10 @@ source $TOP_DIR/openrc
 # Import exercise configuration
 source $TOP_DIR/exerciserc
 
+# If cinder or n-vol are not enabled we exit with exitcode 55 which mean
+# exercise is skipped.
+is_service_enabled cinder n-vol || exit 55
+
 # Instance type to create
 DEFAULT_INSTANCE_TYPE=${DEFAULT_INSTANCE_TYPE:-m1.tiny}
 

files/apts/cinder

@@ -0,0 +1,2 @@
+tgt
+lvm2

files/keystone_data.sh

@@ -9,6 +9,7 @@
 # service              nova      admin, [ResellerAdmin (swift only)]
 # service              quantum   admin        # if enabled
 # service              swift     admin        # if enabled
+# service              cinder    admin        # if enabled
 # demo                 admin     admin
 # demo                 demo      Member, anotherrole
 # invisible_to_admin   demo      Member
@@ -128,3 +129,13 @@ if [[ "$ENABLED_SERVICES" =~ "tempest" ]]; then
                                         --email=alt_demo@example.com)
     keystone user-role-add --user $ALT_DEMO_USER --role $MEMBER_ROLE --tenant_id $ALT_DEMO_TENANT
 fi
+
+if [[ "$ENABLED_SERVICES" =~ "cinder" ]]; then
+    CINDER_USER=$(get_id keystone user-create --name=cinder \
+                                              --pass="$SERVICE_PASSWORD" \
+                                              --tenant_id $SERVICE_TENANT \
+                                              --email=cinder@example.com)
+    keystone user-role-add --tenant_id $SERVICE_TENANT \
+                           --user_id $CINDER_USER \
+                           --role_id $ADMIN_ROLE
+fi

files/rpms/cinder

@@ -0,0 +1,2 @@
+lvm2
+scsi-target-utils

functions

@@ -253,6 +253,7 @@ function is_service_enabled() {
     for service in ${services}; do
         [[ ,${ENABLED_SERVICES}, =~ ,${service}, ]] && return 0
         [[ ${service} == "nova" && ${ENABLED_SERVICES} =~ "n-" ]] && return 0
+        [[ ${service} == "cinder" && ${ENABLED_SERVICES} =~ "c-" ]] && return 0
         [[ ${service} == "glance" && ${ENABLED_SERVICES} =~ "g-" ]] && return 0
         [[ ${service} == "quantum" && ${ENABLED_SERVICES} =~ "q-" ]] && return 0
     done

lib/cinder

@@ -0,0 +1,154 @@
+# lib/cinder
+# Install and start Cinder volume service
+
+# Dependencies:
+# - functions
+# - KEYSTONE_AUTH_* must be defined
+# SERVICE_{TENANT_NAME|PASSWORD} must be defined
+
+# stack.sh
+# ---------
+# install_XXX
+# configure_XXX
+# init_XXX
+# start_XXX
+# stop_XXX
+# cleanup_XXX
+
+# Print the commands being run so that we can see the command that triggers
+# an error.  It is also useful for following along as the install occurs.
+set -o xtrace
+
+
+# Defaults
+# --------
+
+# set up default directories
+CINDER_DIR=$DEST/cinder
+CINDERCLIENT_DIR=$DEST/python-cinderclient
+CINDER_CONF_DIR=/etc/cinder
+CINDER_CONF=$CINDER_CONF_DIR/cinder.conf
+
+# Name of the lvm volume group to use/create for iscsi volumes
+VOLUME_GROUP=${VOLUME_GROUP:-stack-volumes}
+VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}
+
+# cleanup_cinder() - Remove residual data files, anything left over from previous
+# runs that a clean run would need to clean up
+function cleanup_cinder() {
+    # This function intentionally left blank
+    :
+}
+
+# configure_cinder() - Set config files, create data dirs, etc
+function configure_cinder() {
+    setup_develop $CINDER_DIR
+    setup_develop $CINDERCLIENT_DIR
+
+    if [[ ! -d $CINDER_CONF_DIR ]]; then
+        sudo mkdir -p $CINDER_CONF_DIR
+    fi
+    sudo chown `whoami` $CINDER_CONF_DIR
+
+    cp -p $CINDER_DIR/etc/cinder/policy.json $CINDER_CONF_DIR
+
+    CINDER_API_PASTE_INI=$CINDER_CONF_DIR/api-paste.ini
+    cp $CINDER_DIR/etc/cinder/api-paste.ini $CINDER_API_PASTE_INI
+    iniset $CINDER_API_PASTE_INI filter:authtoken auth_host $KEYSTONE_AUTH_HOST
+    iniset $CINDER_API_PASTE_INI filter:authtoken auth_port $KEYSTONE_AUTH_PORT
+    iniset $CINDER_API_PASTE_INI filter:authtoken auth_protocol $KEYSTONE_AUTH_PROTOCOL
+    iniset $CINDER_API_PASTE_INI filter:authtoken admin_tenant_name $SERVICE_TENANT_NAME
+    iniset $CINDER_API_PASTE_INI filter:authtoken admin_user cinder
+    iniset $CINDER_API_PASTE_INI filter:authtoken admin_password $SERVICE_PASSWORD
+
+    cp $CINDER_DIR/etc/cinder/cinder.conf.sample $CINDER_CONF
+    iniset $CINDER_CONF DEFAULT auth_strategy keystone
+    iniset $CINDER_CONF DEFAULT verbose True
+    iniset $CINDER_CONF DEFAULT volume_group $VOLUME_GROUP
+    iniset $CINDER_CONF DEFAULT volume_name_template ${VOLUME_NAME_PREFIX}%s
+    iniset $CINDER_CONF DEFAULT iscsi_helper tgtadm
+    iniset $CINDER_CONF DEFAULT sql_connection $BASE_SQL_CONN/cinder?charset=utf8
+    iniset $CINDER_CONF DEFAULT rabbit_host $RABBIT_HOST
+    iniset $CINDER_CONF DEFAULT rabbit_password $RABBIT_PASSWORD
+    iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI
+}
+
+# init_cinder() - Initialize database and volume group
+function init_cinder() {
+    # Force nova volumes off
+    NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/osapi_volume,//")
+
+    if is_service_enabled mysql; then
+        # (re)create cinder database
+        mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS cinder;'
+        mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE cinder;'
+
+        # (re)create cinder database
+        $CINDER_DIR/bin/cinder-manage db sync
+    fi
+
+    if is_service_enabled c-vol; then
+        # Configure a default volume group called '`stack-volumes`' for the volume
+        # service if it does not yet exist.  If you don't wish to use a file backed
+        # volume group, create your own volume group called ``stack-volumes`` before
+        # invoking ``stack.sh``.
+        #
+        # By default, the backing file is 2G in size, and is stored in ``/opt/stack/data``.
+
+        if ! sudo vgs $VOLUME_GROUP; then
+            VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-$DATA_DIR/${VOLUME_GROUP}-backing-file}
+            VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-2052M}
+            # Only create if the file doesn't already exists
+            [[ -f $VOLUME_BACKING_FILE ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $VOLUME_BACKING_FILE
+            DEV=`sudo losetup -f --show $VOLUME_BACKING_FILE`
+            # Only create if the loopback device doesn't contain $VOLUME_GROUP
+            if ! sudo vgs $VOLUME_GROUP; then sudo vgcreate $VOLUME_GROUP $DEV; fi
+        fi
+
+        if sudo vgs $VOLUME_GROUP; then
+            # Remove iscsi targets
+            sudo tgtadm --op show --mode target | grep $VOLUME_NAME_PREFIX | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
+            # Clean out existing volumes
+            for lv in `sudo lvs --noheadings -o lv_name $VOLUME_GROUP`; do
+                # VOLUME_NAME_PREFIX prefixes the LVs we want
+                if [[ "${lv#$VOLUME_NAME_PREFIX}" != "$lv" ]]; then
+                    sudo lvremove -f $VOLUME_GROUP/$lv
+                fi
+            done
+        fi
+    fi
+}
+
+# install_cinder() - Collect source and prepare
+function install_cinder() {
+    git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH
+    git_clone $CINDERCLIENT_REPO $CINDERCLIENT_DIR $CINDERCLIENT_BRANCH
+}
+
+# start_cinder() - Start running processes, including screen
+function start_cinder() {
+    if is_service_enabled c-vol; then
+        if [[ "$os_PACKAGE" = "deb" ]]; then
+            # tgt in oneiric doesn't restart properly if tgtd isn't running
+            # do it in two steps
+            sudo stop tgt || true
+            sudo start tgt
+        else
+            # bypass redirection to systemctl during restart
+            sudo /sbin/service --skip-redirect tgtd restart
+        fi
+    fi
+
+    screen_it c-api "cd $CINDER_DIR && $CINDER_DIR/bin/cinder-api --config-file $CINDER_CONF"
+    screen_it c-vol "cd $CINDER_DIR && $CINDER_DIR/bin/cinder-volume --config-file $CINDER_CONF"
+    screen_it c-sch "cd $CINDER_DIR && $CINDER_DIR/bin/cinder-scheduler --config-file $CINDER_CONF"
+}
+
+# stop_cinder() - Stop running processes (non-screen)
+function stop_cinder() {
+    # FIXME(dtroyer): stop only the cinder screen window?
+
+    if is_service_enabled c-vol; then
+        stop_service tgt
+    fi
+}

stack.sh

@@ -112,6 +112,13 @@ else
     NOVA_ROOTWRAP=/usr/bin/nova-rootwrap
 fi
 
+# ``stack.sh`` keeps function libraries here
+# Make sure ``$TOP_DIR/lib`` directory is present
+if [ ! -d $TOP_DIR/lib ]; then
+    echo "ERROR: missing devstack/lib - did you grab more than just stack.sh?"
+    exit 1
+fi
+
 # stack.sh keeps the list of ``apt`` and ``pip`` dependencies in external
 # files, along with config templates and other useful files.  You can find these
 # in the ``files`` directory (next to this script).  We will reference this
@@ -130,6 +137,12 @@ if type -p screen >/dev/null && screen -ls | egrep -q "[0-9].stack"; then
     exit 1
 fi
 
+# Make sure we only have one volume service enabled.
+if is_service_enabled cinder && is_service_enabled n-vol; then
+    echo "ERROR: n-vol and cinder must not be enabled at the same time"
+    exit 1
+fi
+
 # OpenStack is designed to be run as a regular user (Horizon will fail to run
 # as root, since apache refused to startup serve content from root user).  If
 # ``stack.sh`` is run as **root**, it automatically creates a **stack** user with
@@ -201,6 +214,19 @@ fi
 # prerequisites and initialize ``$DEST``.
 OFFLINE=`trueorfalse False $OFFLINE`
 
+# Destination path for service data
+DATA_DIR=${DATA_DIR:-${DEST}/data}
+sudo mkdir -p $DATA_DIR
+sudo chown `whoami` $DATA_DIR
+
+
+# Projects
+# --------
+
+# Get project function libraries
+source $TOP_DIR/lib/cinder
+
+
 # Set the destination directories for openstack projects
 NOVA_DIR=$DEST/nova
 HORIZON_DIR=$DEST/horizon
@@ -234,7 +260,7 @@ M_HOST=${M_HOST:-localhost}
 M_MAC_RANGE=${M_MAC_RANGE:-FE-EE-DD-00-00-00/24}
 
 # Name of the lvm volume group to use/create for iscsi volumes
-VOLUME_GROUP=${VOLUME_GROUP:-nova-volumes}
+VOLUME_GROUP=${VOLUME_GROUP:-stack-volumes}
 VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}
 INSTANCE_NAME_PREFIX=${INSTANCE_NAME_PREFIX:-instance-}
 
@@ -607,6 +633,10 @@ function get_packages() {
             if [[ ! $file_to_parse =~ glance ]]; then
                 file_to_parse="${file_to_parse} glance"
             fi
+        elif [[ $service == c-* ]]; then
+            if [[ ! $file_to_parse =~ cinder ]]; then
+                file_to_parse="${file_to_parse} cinder"
+            fi
         elif [[ $service == n-* ]]; then
             if [[ ! $file_to_parse =~ nova ]]; then
                 file_to_parse="${file_to_parse} nova"
@@ -704,10 +734,12 @@ if is_service_enabled m-svc; then
     # melange
     git_clone $MELANGE_REPO $MELANGE_DIR $MELANGE_BRANCH
 fi
-
 if is_service_enabled melange; then
     git_clone $MELANGECLIENT_REPO $MELANGECLIENT_DIR $MELANGECLIENT_BRANCH
 fi
+if is_service_enabled cinder; then
+    install_cinder
+fi
 
 
 # Initialization
@@ -743,6 +775,9 @@ fi
 if is_service_enabled melange; then
     setup_develop $MELANGECLIENT_DIR
 fi
+if is_service_enabled cinder; then
+    configure_cinder
+fi
 
 # Do this _after_ glance is installed to override the old binary
 setup_develop $GLANCECLIENT_DIR
@@ -1643,17 +1678,18 @@ fi
 # Volume Service
 # --------------
 
-if is_service_enabled n-vol; then
+if is_service_enabled cinder; then
-    #
+    init_cinder
-    # Configure a default volume group called 'nova-volumes' for the nova-volume
+elif is_service_enabled n-vol; then
+    # Configure a default volume group called '`stack-volumes`' for the volume
     # service if it does not yet exist.  If you don't wish to use a file backed
-    # volume group, create your own volume group called 'nova-volumes' before
+    # volume group, create your own volume group called ``stack-volumes`` before
-    # invoking stack.sh.
+    # invoking ``stack.sh``.
     #
-    # By default, the backing file is 2G in size, and is stored in /opt/stack.
+    # By default, the backing file is 2G in size, and is stored in ``/opt/stack/data``.
 
     if ! sudo vgs $VOLUME_GROUP; then
-        VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-$DEST/nova-volumes-backing-file}
+        VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-$DATA_DIR/${VOLUME_GROUP}-backing-file}
         VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-2052M}
         # Only create if the file doesn't already exists
         [[ -f $VOLUME_BACKING_FILE ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $VOLUME_BACKING_FILE
@@ -1801,6 +1837,10 @@ else
     add_nova_opt "logging_context_format_string=%(asctime)s %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s"
 fi
 
+# If cinder is enabled, use the cinder volume driver
+if is_service_enabled cinder; then
+    add_nova_opt "volume_api_class=nova.volume.cinder.API"
+fi
 
 # Provide some transition from EXTRA_FLAGS to EXTRA_OPTS
 if [[ -z "$EXTRA_OPTS" && -n "$EXTRA_FLAGS" ]]; then
@@ -1968,6 +2008,7 @@ fi
 
 # launch the nova-api and wait for it to answer before continuing
 if is_service_enabled n-api; then
+    add_nova_opt "enabled_apis=$NOVA_ENABLED_APIS"
     screen_it n-api "cd $NOVA_DIR && $NOVA_DIR/bin/nova-api"
     echo "Waiting for nova-api to start..."
     if ! timeout $SERVICE_TIMEOUT sh -c "while ! http_proxy= wget -q -O- http://127.0.0.1:8774; do sleep 1; done"; then
@@ -2003,6 +2044,9 @@ screen_it n-sch "cd $NOVA_DIR && $NOVA_DIR/bin/nova-scheduler"
 screen_it n-novnc "cd $NOVNC_DIR && ./utils/nova-novncproxy --config-file $NOVA_CONF_DIR/$NOVA_CONF --web ."
 screen_it n-xvnc "cd $NOVA_DIR && ./bin/nova-xvpvncproxy --config-file $NOVA_CONF_DIR/$NOVA_CONF"
 screen_it n-cauth "cd $NOVA_DIR && ./bin/nova-consoleauth"
+if is_service_enabled cinder; then
+    start_cinder
+fi
 screen_it horizon "cd $HORIZON_DIR && sudo tail -f /var/log/$APACHE_NAME/horizon_error.log"
 screen_it swift "cd $SWIFT_DIR && $SWIFT_DIR/bin/swift-proxy-server ${SWIFT_CONFIG_DIR}/proxy-server.conf -v"
 

stackrc

@@ -9,6 +9,17 @@ RC_DIR=$(cd $(dirname "$BASH_SOURCE") && pwd)
 # ENABLED_SERVICES="$ENABLED_SERVICES,swift"
 ENABLED_SERVICES=g-api,g-reg,key,n-api,n-crt,n-obj,n-cpu,n-net,n-vol,n-sch,n-novnc,n-xvnc,n-cauth,horizon,mysql,rabbit
 
+# Set the default Nova APIs to enable
+NOVA_ENABLED_APIS=ec2,osapi_compute,osapi_volume,metadata
+
+# volume service
+CINDER_REPO=https://github.com/openstack/cinder
+CINDER_BRANCH=master
+
+# volume client
+CINDERCLIENT_REPO=https://github.com/openstack/python-cinderclient
+CINDERCLIENT_BRANCH=master
+
 # compute service
 NOVA_REPO=https://github.com/openstack/nova.git
 NOVA_BRANCH=master

unstack.sh

@@ -44,7 +44,7 @@ if is_service_enabled horizon; then
 fi
 
 # Get the iSCSI volumes
-if is_service_enabled n-vol; then
+if is_service_enabled cinder n-vol; then
     TARGETS=$(sudo tgtadm --op show --mode target)
     if [[ -n "$TARGETS" ]]; then
         # FIXME(dtroyer): this could very well require more here to