drbd-devstack/devstack/lib/drbd_devstack

#!/bin/bash
# vim: set et ts=4 sw=4 ft=sh :
#

DRBDMANAGE_DBUS_AUTH_FILE=/etc/dbus-1/system.d/org.drbd.drbdmanaged-stack.conf

function pre_install_drbd_devstack {
    # Install OS packages, if necessary
    if [[ ! -d "${FILES:?FILES not set yet}"  ]]; then
        mkdir "${FILES}"
    fi

    sudo add-apt-repository ppa:linbit/linbit-drbd9-stack

    # install packages normally
    sudo apt-get update
    sudo apt-get install --yes debhelper python-dbus dbus patch \
        lvm2 thin-provisioning-tools drbd-utils drbd-dkms python-drbdmanage \
        linux-headers-$(uname -r)

    # ensure we're starting with upstream config file, ie.
    # overwrite local modifications, so that the automated
    # processes later on work as intended
    sudo apt-get install --reinstall -o Dpkg::Options::=--force-confnew python-drbdmanage

    # Hotfix needed? Avoid dput delay.
    HOTFIXURL=https://gist.githubusercontent.com/haySwim/2c18a8b8510caf7acf99d0d56a602393/raw/daaff7f59bc966cf897d5a527b12ddd37bf4d840/hf.patch
    HOTFIXFILE="${FILES}/drbdhotfix.patch"
    if wget -O "$HOTFIXFILE" "$HOTFIXURL" ; then
        ( cd / ; sudo patch --dry-run --forward --verbose -p0 ) < "$HOTFIXFILE" &&
        ( cd / ; sudo patch           --forward --verbose -p0 ) < "$HOTFIXFILE"
    fi

    return 0
}

function install_drbd_devstack {
    # Install the service.

    # Write DRBDmanage configuration;
    # use the single-thinpool driver for these tests.
    sudo sed -i "s/^drbdctrl-vg\s*=.*/drbdctrl-vg = ${DRBD_DRBDCTRL_VG}/g" \
        /etc/drbdmanaged.cfg
    echo "
[LOCAL]
storage-plugin = drbdmanage.storage.lvm_thinlv.LvmThinLv
force=1

[Plugin:ThinLV]
volume-group = $DRBD_DATA_VG
pool-name = drbdthinpool
    " | sudo tee /etc/drbdmanaged.cfg

    # allow the stack user access to drbdmanage
    sudo tee "$DRBDMANAGE_DBUS_AUTH_FILE" > /dev/null << "EOF"
    <!DOCTYPE busconfig PUBLIC
        "-//freedesktop//DTD D-BUS Bus Configuration 1.0//EN"
        "http://www.freedesktop.org/standards/dbus/1.0/busconfig.dtd">
    <busconfig>
        <policy user="stack">
            <allow send_destination="org.drbd.drbdmanaged"/>
        </policy>
    </busconfig>
EOF

    # done.
}

function _find_lo_for_dev {
    sudo losetup -a | grep "$1"
}

function _drbd_make_vg {
    local vg_name="${1:?No VG name given}"
    local vg_size="$2"
    local vg_lo_dev="$3"

    local vg_dev="/dev/${vg_lo_dev}"

    # If the VG exists, there's nothing left to do.
    if sudo vgdisplay | grep -q "${vg_name}" ; then
        return
    fi

    # if the file exists, don't destroy it...
    if [[ ! -f "${FILES}/${vg_name}" ]]; then
        sudo truncate -s "${vg_size}" "${FILES}/${vg_name}"
    fi

    # if the loop device is present, don't recreate it...
    if [[ ! -e "${vg_dev}" ]]; then
        local vg_lo_minor="$(echo "${vg_lo_dev}" | sed 's/loop//g')"
        sudo mknod -m 660 "${vg_dev}" b 7 "${vg_lo_minor}"
    fi

    # if the file is already assigned a loop device, don't reassign
    if ! _find_lo_for_dev "${vg_lo_dev}" | grep -q "${vg_name}" ; then
        sudo losetup "${vg_dev}" "${FILES}/${vg_name}"
    fi

    local lvm_cfg="devices { global_filter=[ 'a|${vg_lo_dev}|' ] }"

#    # if the lvm.conf already accepts the loop device, don't insert it again
#    if ! sudo grep -q "${vg_lo_dev}" /etc/lvm/lvm.conf ; then
#        sudo sed -i.drbdctrl-bak "s/global_filter = \[ /global_filter = \[ \"a|${vg_lo_dev}|\", /g" /etc/lvm/lvm.conf
#    fi

    # if theres already a pv signature, don't try to recreate
    if ! sudo pvdisplay | grep -q "${vg_lo_dev}" ; then
        sudo pvscan --config "${lvm_cfg}"

        if sudo pvdisplay | grep -q "${vg_lo_dev}" ; then
            sudo pvcreate --config "${lvm_cfg}" "${vg_dev}"
        fi
    fi

    # if theres already a vg, don't try to recreate
    if ! sudo vgdisplay | grep -q "${vg_name}" ; then
        sudo vgscan
        if ! sudo vgdisplay | grep -q "${vg_name}" ; then
            sudo vgcreate --config "${lvm_cfg}" "${vg_name}" "${vg_dev}"
        fi
    fi
}

function _modify_udev_file {
    local file="$1"
    local search="$2"
    local sed_cmd="$3"
    local path="/lib/udev/rules.d/${file}"

    if [[ -f "$path" ]] && ! grep "${search}" "$path"; then
        sudo sed -i "${sed_cmd}" "$path"
    fi
}

function _this_is_the_initial_node {
    [[ -z "$SERVICE_HOST" ||
        "$SERVICE_HOST" == "127.0.0.1" ||
        "$SERVICE_HOST" == "$HOST_IP" ]]
}

function configure_drbd_devstack {
    # Configure the service.
    # This gets called before starting the c-vol service; the next callback,
    # init_drbd_devstack, is too late for that, so we need to make DRBDmanage
    # operational here.
    local be_name="${1:-drbdmanage}"

    # Initialize and start the service.
    # need to setup loopback device(s) for DRBD

    _drbd_make_vg "${DRBD_DRBDCTRL_VG:?DRBD_DRBDCTRL_VG is not set}" \
        "${DRBD_DRBDCTRL_VG_SZ}" "${DRBD_DRBDCTRL_LODEV}"

    # Do the same thing for the DATA volume group
    _drbd_make_vg "${DRBD_DATA_VG:?DRBD_DATA_VG is not set}" \
        "${DRBD_DATA_VG_SZ}" "${DRBD_DATA_LODEV}"

    local vg_size=$(LC_ALL=C sudo vgdisplay --columns  --units M \
        --noheadings -o vg_free --nosuffix "${DRBD_DATA_VG}")
    # No quotes, so that the whitespace around the number gets eaten by the shell
    local thinpool_size=$( echo $vg_size \* 30 / 32 - 64 | bc )

    if ! sudo lvdisplay "${DRBD_DATA_VG}/drbdthinpool" ; then
        sudo /sbin/lvcreate -L "${thinpool_size}"M -T "${DRBD_DATA_VG}/drbdthinpool"
    fi


    # Deactivate udev calling blkid etc. on the DRBD backend devices - that is
    # neither needed nor wanted, because blkid having the device open can race
    # with DRBD attaching.
    # This will result in a message like
    #    kernel: [...] drbd CV_...: drbdX: open("/dev/...") failed with -16
    # which means EBUSY
    _modify_udev_file 60-persistent-storage-dm.rules \
        'ENV{DM_NAME}=="'"${DRBD_DATA_VG}"'-CV_' \
        's:DM_SUSPENDED.*=="1",.*GOTO="\(.*\)":&\nENV{DM_NAME}=="'"${DRBD_DATA_VG}"'-CV_*", GOTO="\1"\n:'
    _modify_udev_file 80-btrfs-lvm.rules \
        'ENV{DM_NAME}=="'"${DRBD_DATA_VG}"'-CV_' \
        's:SUBSYSTEM.="block",.*GOTO="\(.*\)":&\nENV{DM_NAME}=="'"${DRBD_DATA_VG}"'-CV_*", GOTO="\1"\n:'

    # The same can happen to the DRBDmanage control volume, too, of course!
    _modify_udev_file 60-persistent-storage-dm.rules \
        'ENV{DM_NAME}=="'"${DRBD_DRBDCTRL_VG}"'-.drbdctrl' \
        's:DM_SUSPENDED.*=="1",.*GOTO="\(.*\)":&\nENV{DM_NAME}=="'"${DRBD_DRBDCTRL_VG}"'-.drbdctrl", GOTO="\1"\n:'
    _modify_udev_file 80-btrfs-lvm.rules \
        'ENV{DM_NAME}=="'"${DRBD_DRBDCTRL_VG}"'-.drbdctrl' \
        's:SUBSYSTEM.="block",.*GOTO="\(.*\)":&\nENV{DM_NAME}=="'"${DRBD_DRBDCTRL_VG}"'-.drbdctrl", GOTO="\1"\n:'

    _modify_udev_file 69-lvm-metad.rules \
        '#dont run pvscan ## lvm pvscan --cache --activate' \
        's:^RUN+="/sbin/lvm pvscan --cache:# Dont run pvscan:'

    # drbdmanage modii: --no-control-volume --no-storage --no-autojoin


    no_stor="${CINDER_DRBD_NO_STORAGE:+--no-storage}"

    # initialize drbdmanage
    if _this_is_the_initial_node ; then
        # No quotes, so that it can expand to an empty string (and get ignored), too.
        sudo drbdmanage init --quiet $no_stor
    else
        # Do we want $HOST_IP, or a possibly different one that should be reachable by the SERVICE_HOST?
        # Is the $SERVICE_HOST always a drbdmanage node, or should we go to $CINDER_SERVICE_HOST?
        # for now some debug output
        set | grep SERVICE_HOST

        local my_address="$HOST_IP"
        if [[ -z "$my_address" ]] ; then
            # Fallback, in case that script gets called standalone.
            my_address="$(ip -oneline route get "$SERVICE_HOST" | \
                grep "$SERVICE_HOST" | awk '/src (.*)/ { print $5 }')"
        fi
        echo "I am from $my_address" # ;)

        no_cv="${CINDER_DRBD_NO_CV:+--no-control-volume}"

        ssh -oBatchMode=yes -oStrictHostKeyChecking=no stack@"$SERVICE_HOST" \
                sudo drbdmanage new-node --no-autojoin "$no_cv" "$no_stor" \
                    --quiet "$HOSTNAME" "$my_address"

        ssh -oBatchMode=yes -oStrictHostKeyChecking=no stack@"$SERVICE_HOST" \
                sudo drbdmanage howto-join "$HOSTNAME" --quiet | sudo bash
    fi

    sudo drbdmanage shutdown --quiet
    sudo drbdmanage wait-for-startup
    sudo drbdmanage debug 'set loglevel=debug'
    sudo drbdmanage nodes


    if [[ -n "$CINDER_CONF" && -f "$CINDER_CONF" ]] ; then
        iniset $CINDER_CONF "$be_name" volume_backend_name "$be_name"

        local driver=DrbdManageIscsiDriver
        if [[ -n "$CINDER_DRBD_USE_DRBD_PROTOCOL" ]] ; then
            driver=DrbdManageDrbdDriver
        fi
        iniset $CINDER_CONF "$be_name" volume_driver \
            cinder.volume.drivers.drbdmanagedrv.$driver
    fi
}

function init_drbd_devstack {
    true
}

function shutdown_drbd_devstack {
    # Shut the service down.
    # drbdadm down all
    # drbdmanage shutdown --quiet
    echo "shutdown drbd devstack"
}

function cleanup_drbd_devstack {
    # Cleanup the service.
    # something like
    #     drbdmanage list-resource --short | xargs -l drbdmanage remove-resource
    # ???
    # drbdmanage resources -m | sed 's/,.*//g' | xargs -l drbdmanage remove-resource --quiet
    echo "cleanup drbd devstack"
}

#debug main
#source $(dirname '$0')/../settings
#pre_install_drbd_devstack
#install_drbd_devstack
#configure_drbd_devstack
#init_drbd_devstack

# Tell emacs to use shell-script-mode
## Local variables:
## mode: shell-script
## End: