From 4bf861c76c220a98a3b3165eea5448411d000f3a Mon Sep 17 00:00:00 2001 From: John Griffith Date: Tue, 17 Mar 2015 21:07:39 -0600 Subject: [PATCH] Create global_filter to avoid scan of missing devs A while back I added an lvm.conf file with a device filter setting to try and clean up the LVM hangs in the gate: (commit 0b9e76f280208b5b5ad54bb6fbc4133e63037286) It turns out this wasn't the real problem, the real problem is that on an LVS/VGS command LVM will attempt to open and read all potential block devices in /dev to see if they have LVM data on them. I initially thought the local filter would keep that from happening, as it turns out the local filter only limits what's returned AFTER the actual scan process. In order to keep the scan from happening at all, either a global_filter needs to be used or lvmetad needs to be running and enabled. There are situations in gate tests where /dev/sdX devices are created and deleted and the result is that we hit situations where LVM tries to open up devices to check them even if they've been removed. The result is we have a blocking open call from LVM that takes approx 60 seconds to time out and fail. Ubuntu won't have a version of lvmetad until Vivid, so for now that just leaves the global_filter as an option. This patch adds the filter routine to the end of stack.sh. We don't want to put the routine in lib/cinder_backend/lvm like we had it because now we have to set the global filter for all LVM commands on the system. So we put this as one of the last steps in stack.sh and run it if Cinder is enabled. This way we can query PV's on the system regardless of what other services may be running and using LVM and make sure that all of their devices are added to the filter as well. Also, make sure we only set this for Ubuntu as Fedora/RHEL variants utilize lvmetad. This patch also removes the old change that set the local filter. DocImpact Should add this to recommended config for Cinder on systems that don't have lvmetad, and recommend lvmetad for those that do. Change-Id: I5d5c48e188cbb9b4208096736807f082bce524e8 Closes-Bug: #1373513 --- lib/cinder | 17 ----------------- lib/cinder_backends/lvm | 31 ------------------------------- lib/lvm | 25 +++++++++++++++++++++++++ stack.sh | 9 +++++++++ 4 files changed, 34 insertions(+), 48 deletions(-) diff --git a/lib/cinder b/lib/cinder index 880af1fd40..958c7f05ac 100644 --- a/lib/cinder +++ b/lib/cinder @@ -372,15 +372,9 @@ function init_cinder { if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then local be be_name be_type - local has_lvm=0 for be in ${CINDER_ENABLED_BACKENDS//,/ }; do be_type=${be%%:*} be_name=${be##*:} - - if [[ $be_type == 'lvm' ]]; then - has_lvm=1 - fi - if type init_cinder_backend_${be_type} >/dev/null 2>&1; then # Always init the default volume group for lvm. if [[ "$be_type" == "lvm" ]]; then @@ -391,17 +385,6 @@ function init_cinder { done fi - # Keep it simple, set a marker if there's an LVM backend - # use the created VG's to setup lvm filters - if [[ $has_lvm == 1 ]]; then - # Order matters here, not only obviously to make - # sure the VG's are created, but also some distros - # do some customizations to lvm.conf on init, we - # want to make sure we copy those over - sudo cp /etc/lvm/lvm.conf /etc/cinder/lvm.conf - configure_cinder_backend_conf_lvm - fi - mkdir -p $CINDER_STATE_PATH/volumes create_cinder_cache_dir } diff --git a/lib/cinder_backends/lvm b/lib/cinder_backends/lvm index 52fc6fbf63..f210578339 100644 --- a/lib/cinder_backends/lvm +++ b/lib/cinder_backends/lvm @@ -19,7 +19,6 @@ # clean_cinder_backend_lvm - called from clean_cinder() # configure_cinder_backend_lvm - called from configure_cinder() # init_cinder_backend_lvm - called from init_cinder() -# configure_cinder_backend_conf_lvm - called from configure_cinder() # Save trace setting @@ -66,36 +65,6 @@ function init_cinder_backend_lvm { init_lvm_volume_group $VOLUME_GROUP_NAME-$be_name $VOLUME_BACKING_FILE_SIZE } -# configure_cinder_backend_conf_lvm - Sets device filter in /etc/cinder/lvm.conf -# init_cinder_backend_lvm -function configure_cinder_backend_conf_lvm { - local filter_suffix='"r/.*/" ]' - local filter_string="filter = [ " - local conf_entries=$(grep volume_group /etc/cinder/cinder.conf | sed "s/ //g") - local pv - local vg - local line - - for pv_info in $(sudo pvs --noheadings -o name,vg_name --separator ';'); do - echo_summary "Evaluate PV info for Cinder lvm.conf: $pv_info" - IFS=';' read pv vg <<< "$pv_info" - for line in ${conf_entries}; do - IFS='=' read label group <<< "$line" - group=$(echo $group|sed "s/^ *//g") - if [[ "$vg" == "$group" ]]; then - new="\"a$pv/\", " - filter_string=$filter_string$new - fi - done - done - filter_string=$filter_string$filter_suffix - - # FIXME(jdg): Possible odd case that the lvm.conf file has been modified - # and doesn't have a filter entry to search/replace. For devstack don't - # know that we care, but could consider adding a check and add - sudo sed -i "s#^[ \t]*filter.*# $filter_string#g" /etc/cinder/lvm.conf - echo "set LVM filter_strings: $filter_string" -} # Restore xtrace $MY_XTRACE diff --git a/lib/lvm b/lib/lvm index 39eed00675..d0322c76b3 100644 --- a/lib/lvm +++ b/lib/lvm @@ -138,6 +138,31 @@ function init_default_lvm_volume_group { fi } +# set_lvm_filter() Gather all devices configured for LVM and +# use them to build a global device filter +# set_lvm_filter() Create a device filter +# and add to /etc/lvm.conf. Note this uses +# all current PV's in use by LVM on the +# system to build it's filter. +# +# Usage: set_lvm_filter() +function set_lvm_filter { + local filter_suffix='"r|.*|" ]' + local filter_string="global_filter = [ " + local pv + local vg + local line + + for pv_info in $(sudo pvs --noheadings -o name); do + pv=$(echo -e "${pv_info}" | sed 's/ //g' | sed 's/\/dev\///g') + new="\"a|$pv|\", " + filter_string=$filter_string$new + done + filter_string=$filter_string$filter_suffix + + sudo sed -i "/# global_filter = \[*\]/a\ $global_filter$filter_string" /etc/lvm/lvm.conf + echo_summary "set lvm.conf device global_filter to: $filter_string" +} # Restore xtrace $MY_XTRACE diff --git a/stack.sh b/stack.sh index eac7eec724..ee1b985eab 100755 --- a/stack.sh +++ b/stack.sh @@ -1316,6 +1316,15 @@ service_check # Prepare bash completion for OSC openstack complete | sudo tee /etc/bash_completion.d/osc.bash_completion > /dev/null +# If cinder is configured, set global_filter for PV devices +if is_service_enabled cinder; then + if is_ubuntu; then + echo_summary "Configuring lvm.conf global device filter" + set_lvm_filter + else + echo_summary "Skip setting lvm filters for non Ubuntu systems" + fi +fi # Fin # ===