Allow DevStack to retry DNS queries to handle async CRUD
With pool-manager, records are created/deleted asynchronously, so we update the DevStack code to implement a retry+timeout on DNS queries. Additionally, move exercises only functions into the exercises script. Change-Id: Ic94fac4c4a2845780c24e9489d39b4f2f39a1d55 Closes-Bug: 1398786
This commit is contained in:
parent
33866dfbd1
commit
6df939f2a3
@ -17,9 +17,6 @@ set -o errexit
|
|||||||
set -o xtrace
|
set -o xtrace
|
||||||
|
|
||||||
|
|
||||||
# Settings
|
|
||||||
# ========
|
|
||||||
|
|
||||||
# Keep track of the current directory
|
# Keep track of the current directory
|
||||||
EXERCISE_DIR=$(cd $(dirname "$0") && pwd)
|
EXERCISE_DIR=$(cd $(dirname "$0") && pwd)
|
||||||
TOP_DIR=$(cd $EXERCISE_DIR/..; pwd)
|
TOP_DIR=$(cd $EXERCISE_DIR/..; pwd)
|
||||||
@ -39,10 +36,128 @@ is_service_enabled designate || exit 55
|
|||||||
# Import designate library
|
# Import designate library
|
||||||
source $TOP_DIR/lib/designate
|
source $TOP_DIR/lib/designate
|
||||||
|
|
||||||
|
# Settings
|
||||||
|
# ========
|
||||||
|
|
||||||
|
# used with dig to look up in DNS
|
||||||
|
DIG_TIMEOUT=30
|
||||||
|
|
||||||
|
# used with dig to look up in DNS
|
||||||
|
DIG_FLAGS="-p $DESIGNATE_SERVICE_PORT_DNS @$DESIGNATE_SERVICE_HOST"
|
||||||
|
|
||||||
|
# used with dig to do an AXFR against MDNS
|
||||||
|
DIG_AXFR_FLAGS="-p $DESIGNATE_SERVICE_PORT_MDNS @$DESIGNATE_SERVICE_HOST AXFR +tcp +nocmd"
|
||||||
|
|
||||||
# NUMBER_OF_RECORDS keeps track of the records we need to get for AXFR
|
# NUMBER_OF_RECORDS keeps track of the records we need to get for AXFR
|
||||||
# We start with 1 to account for the additional SOA at the end
|
# We start with 1 to account for the additional SOA at the end
|
||||||
NUMBER_OF_RECORDS=1
|
NUMBER_OF_RECORDS=1
|
||||||
|
|
||||||
|
# Functions
|
||||||
|
# =========
|
||||||
|
|
||||||
|
function ensure_record_present {
|
||||||
|
local record_name=$1
|
||||||
|
local record_type=$2
|
||||||
|
local record_value=$3
|
||||||
|
|
||||||
|
if [ "$DESIGNATE_BACKEND_DRIVER" = "fake" ] ; then
|
||||||
|
# if the backend is fake, there will be no actual DNS records
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! timeout $DIG_TIMEOUT sh -c "while ! dig +short $DIG_FLAGS $record_name $record_type | grep \"$record_value\"; do sleep 1; done"; then
|
||||||
|
die $LINENO "Error: record $record_name ($record_type) not found in DNS"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Display for debugging
|
||||||
|
dig $DIG_FLAGS $record_name $record_type
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensure_record_absent {
|
||||||
|
local record_name=$1
|
||||||
|
local record_type=$2
|
||||||
|
local record_value=$3
|
||||||
|
|
||||||
|
if [ "$DESIGNATE_BACKEND_DRIVER" = "fake" ] ; then
|
||||||
|
# if the backend is fake, there will be no actual DNS records
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! timeout $DIG_TIMEOUT sh -c "while dig +short $DIG_FLAGS $record_name $record_type | grep \"$record_value\"; do sleep 1; done"; then
|
||||||
|
# Display for debugging
|
||||||
|
dig $DIG_FLAGS $record_name $record_type
|
||||||
|
|
||||||
|
die $LINENO "Error: record $record_name ($record_type) found in DNS, should be absent"
|
||||||
|
fi
|
||||||
|
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# do an AXFR request to MDNS
|
||||||
|
# if it does not match the expected value, give an error
|
||||||
|
function verify_axfr_in_mdns {
|
||||||
|
# Display for debugging
|
||||||
|
dig $DIG_AXFR_FLAGS "$1"
|
||||||
|
if dig $DIG_AXFR_FLAGS "$1"; then
|
||||||
|
if [ -n "$2" ] ; then
|
||||||
|
local axfr_records=$(dig $DIG_AXFR_FLAGS "$1" | grep "$1" | wc -l)
|
||||||
|
if [ "$axfr_records" = "$2" ] ; then
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
die $LINENO "Error: AXFR to MDNS did not return the expected number of records"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
die $LINENO "Error: AXFR to MDNS did not return a correct response"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# get the domain id (uuid) given the domain name
|
||||||
|
# if REQUIRED is set, die with an error if name not found
|
||||||
|
function get_domain_id {
|
||||||
|
local domain_name=$1
|
||||||
|
local required=$2
|
||||||
|
local domain_id=$(designate domain-list | egrep " $domain_name " | get_field 1)
|
||||||
|
if [ "$required" = "1" ] ; then
|
||||||
|
die_if_not_set $LINENO domain_id "Failure retrieving DOMAIN_ID"
|
||||||
|
fi
|
||||||
|
echo "$domain_id"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# get the domain_name given the id
|
||||||
|
function get_domain_name {
|
||||||
|
designate domain-list | grep "$1" | get_field 2
|
||||||
|
}
|
||||||
|
|
||||||
|
# if the given domain does not exist, it will be created
|
||||||
|
# the domain_id of the domain will be returned
|
||||||
|
function get_or_create_domain_id {
|
||||||
|
local domainid=$(get_domain_id "$1")
|
||||||
|
if [[ -z "$domainid" ]]; then
|
||||||
|
designate domain-create --name $1 --email admin@devstack.org --ttl 86400 --description "domain $1" 1>&2
|
||||||
|
domainid=$(designate domain-list | grep "$1" | get_field 1)
|
||||||
|
fi
|
||||||
|
echo $domainid
|
||||||
|
}
|
||||||
|
|
||||||
|
# get the record id (uuid) given the record name and domain id
|
||||||
|
# if REQUIRED is set, die with an error if name not found
|
||||||
|
function get_record_id {
|
||||||
|
local domain_id=$1
|
||||||
|
local record_name=$2
|
||||||
|
local record_type=$3
|
||||||
|
local required=$4
|
||||||
|
local record_id=$(designate record-list $domain_id | egrep " $record_name " | egrep " $record_type " | get_field 1)
|
||||||
|
if [ "$required" = "1" ] ; then
|
||||||
|
die_if_not_set $LINENO record_id "Failure retrieving RECORD_ID"
|
||||||
|
fi
|
||||||
|
echo "$record_id"
|
||||||
|
}
|
||||||
|
|
||||||
# Testing Servers
|
# Testing Servers
|
||||||
# ===============
|
# ===============
|
||||||
designate server-list
|
designate server-list
|
||||||
@ -60,8 +175,8 @@ DOMAIN_NAME="exercise-$(openssl rand -hex 4).com."
|
|||||||
designate domain-create --name $DOMAIN_NAME --email devstack@example.org
|
designate domain-create --name $DOMAIN_NAME --email devstack@example.org
|
||||||
((NUMBER_OF_RECORDS+=2))
|
((NUMBER_OF_RECORDS+=2))
|
||||||
# should have SOA and NS records
|
# should have SOA and NS records
|
||||||
verify_name_type_dns $DOMAIN_NAME SOA $DESIGNATE_TEST_NSREC
|
ensure_record_present $DOMAIN_NAME SOA $DESIGNATE_TEST_NSREC
|
||||||
verify_name_type_dns $DOMAIN_NAME NS $DESIGNATE_TEST_NSREC
|
ensure_record_present $DOMAIN_NAME NS $DESIGNATE_TEST_NSREC
|
||||||
|
|
||||||
DOMAIN_ID=$(get_domain_id $DOMAIN_NAME 1)
|
DOMAIN_ID=$(get_domain_id $DOMAIN_NAME 1)
|
||||||
|
|
||||||
@ -86,7 +201,7 @@ A_RECORD_ID=$(get_record_id $DOMAIN_ID $A_RECORD_NAME A)
|
|||||||
designate record-get $DOMAIN_ID $A_RECORD_ID
|
designate record-get $DOMAIN_ID $A_RECORD_ID
|
||||||
|
|
||||||
# Verify the record is published in DNS
|
# Verify the record is published in DNS
|
||||||
verify_name_type_dns $A_RECORD_NAME A 127.0.0.1
|
ensure_record_present $A_RECORD_NAME A 127.0.0.1
|
||||||
|
|
||||||
# -----
|
# -----
|
||||||
|
|
||||||
@ -102,7 +217,7 @@ AAAA_RECORD_ID=$(get_record_id $DOMAIN_ID $AAAA_RECORD_NAME AAAA)
|
|||||||
designate record-get $DOMAIN_ID $AAAA_RECORD_ID
|
designate record-get $DOMAIN_ID $AAAA_RECORD_ID
|
||||||
|
|
||||||
# Verify the record is published in DNS
|
# Verify the record is published in DNS
|
||||||
verify_name_type_dns $AAAA_RECORD_NAME AAAA 2607:f0d0:1002:51::4
|
ensure_record_present $AAAA_RECORD_NAME AAAA 2607:f0d0:1002:51::4
|
||||||
|
|
||||||
# -----
|
# -----
|
||||||
|
|
||||||
@ -115,7 +230,7 @@ MX_RECORD_ID=$(get_record_id $DOMAIN_ID $DOMAIN_NAME MX)
|
|||||||
designate record-get $DOMAIN_ID $MX_RECORD_ID
|
designate record-get $DOMAIN_ID $MX_RECORD_ID
|
||||||
|
|
||||||
# Verify the record is published in DNS
|
# Verify the record is published in DNS
|
||||||
verify_name_type_dns $DOMAIN_NAME MX "5 mail.example.com."
|
ensure_record_present $DOMAIN_NAME MX "5 mail.example.com."
|
||||||
|
|
||||||
# -----
|
# -----
|
||||||
|
|
||||||
@ -128,7 +243,7 @@ SRV_RECORD_ID=$(get_record_id $DOMAIN_ID _sip._tcp.$DOMAIN_NAME SRV)
|
|||||||
designate record-get $DOMAIN_ID $SRV_RECORD_ID
|
designate record-get $DOMAIN_ID $SRV_RECORD_ID
|
||||||
|
|
||||||
# Verify the record is published in DNS
|
# Verify the record is published in DNS
|
||||||
verify_name_type_dns _sip._tcp.$DOMAIN_NAME SRV "10 5 5060 sip.example.com."
|
ensure_record_present _sip._tcp.$DOMAIN_NAME SRV "10 5 5060 sip.example.com."
|
||||||
|
|
||||||
# -----
|
# -----
|
||||||
|
|
||||||
@ -144,7 +259,7 @@ CNAME_RECORD_ID=$(get_record_id $DOMAIN_ID $CNAME_RECORD_NAME CNAME)
|
|||||||
designate record-get $DOMAIN_ID $CNAME_RECORD_ID
|
designate record-get $DOMAIN_ID $CNAME_RECORD_ID
|
||||||
|
|
||||||
# Verify the record is published in DNS
|
# Verify the record is published in DNS
|
||||||
verify_name_type_dns $CNAME_RECORD_NAME CNAME $DOMAIN_NAME
|
ensure_record_present $CNAME_RECORD_NAME CNAME $DOMAIN_NAME
|
||||||
|
|
||||||
# -----
|
# -----
|
||||||
|
|
||||||
@ -166,7 +281,7 @@ designate record-list $DOMAIN_ID
|
|||||||
designate record-get $DOMAIN_ID $CNAME_RECORD_ID || echo "good - record was removed"
|
designate record-get $DOMAIN_ID $CNAME_RECORD_ID || echo "good - record was removed"
|
||||||
|
|
||||||
# verify not in DNS anymore
|
# verify not in DNS anymore
|
||||||
verify_name_type_dns $CNAME_RECORD_NAME CNAME $DOMAIN_NAME 1
|
ensure_record_absent $CNAME_RECORD_NAME CNAME $DOMAIN_NAME
|
||||||
|
|
||||||
# Testing Domains Delete
|
# Testing Domains Delete
|
||||||
# ======================
|
# ======================
|
||||||
@ -178,8 +293,8 @@ designate domain-delete $DOMAIN_ID
|
|||||||
designate domain-get $DOMAIN_ID || echo "good - domain was removed"
|
designate domain-get $DOMAIN_ID || echo "good - domain was removed"
|
||||||
|
|
||||||
# should not have SOA and NS records
|
# should not have SOA and NS records
|
||||||
verify_name_type_dns $DOMAIN_NAME SOA $DESIGNATE_TEST_NSREC 1
|
ensure_record_absent $DOMAIN_NAME SOA $DESIGNATE_TEST_NSREC
|
||||||
verify_name_type_dns $DOMAIN_NAME NS $DESIGNATE_TEST_NSREC 1
|
ensure_record_absent $DOMAIN_NAME NS $DESIGNATE_TEST_NSREC
|
||||||
|
|
||||||
set +o xtrace
|
set +o xtrace
|
||||||
echo "*********************************************************************"
|
echo "*********************************************************************"
|
||||||
|
@ -54,12 +54,6 @@ DESIGNATE_SERVICE_PORT_DNS=${DESIGNATE_SERVICE_PORT_DNS:-53}
|
|||||||
DESIGNATE_SERVICE_PORT_MDNS=${DESIGNATE_SERVICE_PORT_MDNS:-5354}
|
DESIGNATE_SERVICE_PORT_MDNS=${DESIGNATE_SERVICE_PORT_MDNS:-5354}
|
||||||
DESIGNATE_TEST_NSREC=${DESIGNATE_TEST_NSREC:-ns1.devstack.org.}
|
DESIGNATE_TEST_NSREC=${DESIGNATE_TEST_NSREC:-ns1.devstack.org.}
|
||||||
|
|
||||||
# used with dig to look up in DNS
|
|
||||||
DESIGNATE_DIG_FLAGS="-p $DESIGNATE_SERVICE_PORT_DNS @$DESIGNATE_SERVICE_HOST"
|
|
||||||
|
|
||||||
# used with dig to do an AXFR against MDNS
|
|
||||||
DESIGNATE_DIG_AXFR_FLAGS="-p $DESIGNATE_SERVICE_PORT_MDNS @$DESIGNATE_SERVICE_HOST AXFR +tcp +nocmd"
|
|
||||||
|
|
||||||
DESIGNATE_ENABLED_EXTENSIONS_V1=${DESIGNATE_ENABLED_EXTENSIONS_V1:-"quotas"}
|
DESIGNATE_ENABLED_EXTENSIONS_V1=${DESIGNATE_ENABLED_EXTENSIONS_V1:-"quotas"}
|
||||||
DESIGNATE_ENABLED_EXTENSIONS_V2=${DESIGNATE_ENABLED_EXTENSIONS_V2:-"quotas"}
|
DESIGNATE_ENABLED_EXTENSIONS_V2=${DESIGNATE_ENABLED_EXTENSIONS_V2:-"quotas"}
|
||||||
|
|
||||||
@ -70,99 +64,6 @@ if is_service_enabled designate && [[ -r $DESIGNATE_PLUGINS/backend-$DESIGNATE_B
|
|||||||
source $DESIGNATE_PLUGINS/backend-$DESIGNATE_BACKEND_DRIVER
|
source $DESIGNATE_PLUGINS/backend-$DESIGNATE_BACKEND_DRIVER
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Functions
|
|
||||||
# ---------
|
|
||||||
|
|
||||||
# lookup the given name and type in DNS
|
|
||||||
# if it does not match the expected value, give an error
|
|
||||||
# if $4 is given, reverse the test - assert the value is not present
|
|
||||||
function verify_name_type_dns {
|
|
||||||
if [ "$DESIGNATE_BACKEND_DRIVER" = "fake" ] ; then
|
|
||||||
# if the backend is fake, there will be no actual DNS records
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# give DNS changes time to show up
|
|
||||||
sleep 1
|
|
||||||
|
|
||||||
# Display for debugging
|
|
||||||
dig $DESIGNATE_DIG_FLAGS "$1" "$2"
|
|
||||||
if [ -n "$4" ] ; then
|
|
||||||
if dig +short $DESIGNATE_DIG_FLAGS "$1" "$2" | grep "$3"; then
|
|
||||||
die $LINENO "Error: record $3 found in DNS, should have been removed"
|
|
||||||
fi
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
if dig +short $DESIGNATE_DIG_FLAGS "$1" "$2" | grep "$3"; then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
die $LINENO "Error: record $3 not found in DNS"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# do an AXFR request to MDNS
|
|
||||||
# if it does not match the expected value, give an error
|
|
||||||
function verify_axfr_in_mdns {
|
|
||||||
# Display for debugging
|
|
||||||
dig $DESIGNATE_DIG_AXFR_FLAGS "$1"
|
|
||||||
if dig $DESIGNATE_DIG_AXFR_FLAGS "$1"; then
|
|
||||||
if [ -n "$2" ] ; then
|
|
||||||
local axfr_records=$(dig $DESIGNATE_DIG_AXFR_FLAGS "$1" | grep "$1" | wc -l)
|
|
||||||
if [ "$axfr_records" = "$2" ] ; then
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
die $LINENO "Error: AXFR to MDNS did not return the expected number of records"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
die $LINENO "Error: AXFR to MDNS did not return a correct response"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# get the domain id (uuid) given the domain name
|
|
||||||
# if REQUIRED is set, die with an error if name not found
|
|
||||||
function get_domain_id {
|
|
||||||
local domain_name=$1
|
|
||||||
local required=$2
|
|
||||||
local domain_id=$(designate domain-list | egrep " $domain_name " | get_field 1)
|
|
||||||
if [ "$required" = "1" ] ; then
|
|
||||||
die_if_not_set $LINENO domain_id "Failure retrieving DOMAIN_ID"
|
|
||||||
fi
|
|
||||||
echo "$domain_id"
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# get the domain_name given the id
|
|
||||||
function get_domain_name {
|
|
||||||
designate domain-list | grep "$1" | get_field 2
|
|
||||||
}
|
|
||||||
|
|
||||||
# if the given domain does not exist, it will be created
|
|
||||||
# the domain_id of the domain will be returned
|
|
||||||
function get_or_create_domain_id {
|
|
||||||
local domainid=$(get_domain_id "$1")
|
|
||||||
if [[ -z "$domainid" ]]; then
|
|
||||||
designate domain-create --name $1 --email admin@devstack.org --ttl 86400 --description "domain $1" 1>&2
|
|
||||||
domainid=$(designate domain-list | grep "$1" | get_field 1)
|
|
||||||
fi
|
|
||||||
echo $domainid
|
|
||||||
}
|
|
||||||
|
|
||||||
# get the record id (uuid) given the record name and domain id
|
|
||||||
# if REQUIRED is set, die with an error if name not found
|
|
||||||
function get_record_id {
|
|
||||||
local domain_id=$1
|
|
||||||
local record_name=$2
|
|
||||||
local record_type=$3
|
|
||||||
local required=$4
|
|
||||||
local record_id=$(designate record-list $domain_id | egrep " $record_name " | egrep " $record_type " | get_field 1)
|
|
||||||
if [ "$required" = "1" ] ; then
|
|
||||||
die_if_not_set $LINENO record_id "Failure retrieving RECORD_ID"
|
|
||||||
fi
|
|
||||||
echo "$record_id"
|
|
||||||
}
|
|
||||||
|
|
||||||
# cleanup_designate - Remove residual data files, anything left over from previous
|
# cleanup_designate - Remove residual data files, anything left over from previous
|
||||||
# runs that a clean run would need to clean up
|
# runs that a clean run would need to clean up
|
||||||
function cleanup_designate {
|
function cleanup_designate {
|
||||||
|
Loading…
Reference in New Issue
Block a user