From 6df939f2a308a2864b695e56dd833a7189e0971c Mon Sep 17 00:00:00 2001
From: Kiall Mac Innes <kiall@hp.com>
Date: Wed, 3 Dec 2014 12:40:36 +0000
Subject: [PATCH] Allow DevStack to retry DNS queries to handle async CRUD

With pool-manager, records are created/deleted asynchronously, so we update
the DevStack code to implement a retry+timeout on DNS queries.

Additionally, move exercises only functions into the exercises script.

Change-Id: Ic94fac4c4a2845780c24e9489d39b4f2f39a1d55
Closes-Bug: 1398786
---
 contrib/devstack/exercises/designate.sh | 141 +++++++++++++++++++++---
 contrib/devstack/lib/designate          |  99 -----------------
 2 files changed, 128 insertions(+), 112 deletions(-)

diff --git a/contrib/devstack/exercises/designate.sh b/contrib/devstack/exercises/designate.sh
index 3f67d2ac..daa53baf 100755
--- a/contrib/devstack/exercises/designate.sh
+++ b/contrib/devstack/exercises/designate.sh
@@ -17,9 +17,6 @@ set -o errexit
 set -o xtrace
 
 
-# Settings
-# ========
-
 # Keep track of the current directory
 EXERCISE_DIR=$(cd $(dirname "$0") && pwd)
 TOP_DIR=$(cd $EXERCISE_DIR/..; pwd)
@@ -39,10 +36,128 @@ is_service_enabled designate || exit 55
 # Import designate library
 source $TOP_DIR/lib/designate
 
+# Settings
+# ========
+
+# used with dig to look up in DNS
+DIG_TIMEOUT=30
+
+# used with dig to look up in DNS
+DIG_FLAGS="-p $DESIGNATE_SERVICE_PORT_DNS @$DESIGNATE_SERVICE_HOST"
+
+# used with dig to do an AXFR against MDNS
+DIG_AXFR_FLAGS="-p $DESIGNATE_SERVICE_PORT_MDNS @$DESIGNATE_SERVICE_HOST AXFR +tcp +nocmd"
+
 # NUMBER_OF_RECORDS keeps track of the records we need to get for AXFR
 # We start with 1 to account for the additional SOA at the end
 NUMBER_OF_RECORDS=1
 
+# Functions
+# =========
+
+function ensure_record_present {
+    local record_name=$1
+    local record_type=$2
+    local record_value=$3
+
+    if [ "$DESIGNATE_BACKEND_DRIVER" = "fake" ] ; then
+        # if the backend is fake, there will be no actual DNS records
+        return 0
+    fi
+
+    if ! timeout $DIG_TIMEOUT sh -c "while ! dig +short $DIG_FLAGS $record_name $record_type | grep \"$record_value\"; do sleep 1; done"; then
+        die $LINENO "Error: record $record_name ($record_type) not found in DNS"
+    fi
+
+    # Display for debugging
+    dig $DIG_FLAGS $record_name $record_type
+
+    return 0
+}
+
+function ensure_record_absent {
+    local record_name=$1
+    local record_type=$2
+    local record_value=$3
+
+    if [ "$DESIGNATE_BACKEND_DRIVER" = "fake" ] ; then
+        # if the backend is fake, there will be no actual DNS records
+        return 0
+    fi
+
+    if ! timeout $DIG_TIMEOUT sh -c "while dig +short $DIG_FLAGS $record_name $record_type | grep \"$record_value\"; do sleep 1; done"; then
+        # Display for debugging
+        dig $DIG_FLAGS $record_name $record_type
+
+        die $LINENO "Error: record $record_name ($record_type) found in DNS, should be absent"
+    fi
+
+    return 0
+}
+
+# do an AXFR request to MDNS
+# if it does not match the expected value, give an error
+function verify_axfr_in_mdns {
+    # Display for debugging
+    dig $DIG_AXFR_FLAGS "$1"
+    if dig $DIG_AXFR_FLAGS "$1"; then
+        if [ -n "$2" ] ; then
+            local axfr_records=$(dig $DIG_AXFR_FLAGS "$1" | grep "$1" | wc -l)
+            if [ "$axfr_records" = "$2" ] ; then
+                return 0
+            else
+                die $LINENO "Error: AXFR to MDNS did not return the expected number of records"
+            fi
+        fi
+        return 0
+    else
+        die $LINENO "Error: AXFR to MDNS did not return a correct response"
+    fi
+}
+
+# get the domain id (uuid) given the domain name
+# if REQUIRED is set, die with an error if name not found
+function get_domain_id {
+    local domain_name=$1
+    local required=$2
+    local domain_id=$(designate domain-list | egrep " $domain_name " | get_field 1)
+    if [ "$required" = "1" ] ; then
+        die_if_not_set $LINENO domain_id "Failure retrieving DOMAIN_ID"
+    fi
+    echo "$domain_id"
+}
+
+
+# get the domain_name given the id
+function get_domain_name {
+    designate domain-list | grep "$1" | get_field 2
+}
+
+# if the given domain does not exist, it will be created
+# the domain_id of the domain will be returned
+function get_or_create_domain_id {
+    local domainid=$(get_domain_id "$1")
+    if [[ -z "$domainid" ]]; then
+        designate domain-create --name $1 --email admin@devstack.org --ttl 86400 --description "domain $1" 1>&2
+        domainid=$(designate domain-list | grep "$1" | get_field 1)
+    fi
+    echo $domainid
+}
+
+# get the record id (uuid) given the record name and domain id
+# if REQUIRED is set, die with an error if name not found
+function get_record_id {
+    local domain_id=$1
+    local record_name=$2
+    local record_type=$3
+    local required=$4
+    local record_id=$(designate record-list $domain_id | egrep " $record_name " | egrep " $record_type " | get_field 1)
+    if [ "$required" = "1" ] ; then
+        die_if_not_set $LINENO record_id "Failure retrieving RECORD_ID"
+    fi
+    echo "$record_id"
+}
+
 # Testing Servers
 # ===============
 designate server-list
@@ -60,8 +175,8 @@ DOMAIN_NAME="exercise-$(openssl rand -hex 4).com."
 designate domain-create --name $DOMAIN_NAME --email devstack@example.org
 ((NUMBER_OF_RECORDS+=2))
 # should have SOA and NS records
-verify_name_type_dns $DOMAIN_NAME SOA $DESIGNATE_TEST_NSREC
-verify_name_type_dns $DOMAIN_NAME NS $DESIGNATE_TEST_NSREC
+ensure_record_present $DOMAIN_NAME SOA $DESIGNATE_TEST_NSREC
+ensure_record_present $DOMAIN_NAME NS $DESIGNATE_TEST_NSREC
 
 DOMAIN_ID=$(get_domain_id $DOMAIN_NAME 1)
 
@@ -86,7 +201,7 @@ A_RECORD_ID=$(get_record_id $DOMAIN_ID $A_RECORD_NAME A)
 designate record-get $DOMAIN_ID $A_RECORD_ID
 
 # Verify the record is published in DNS
-verify_name_type_dns $A_RECORD_NAME A 127.0.0.1
+ensure_record_present $A_RECORD_NAME A 127.0.0.1
 
 # -----
 
@@ -102,7 +217,7 @@ AAAA_RECORD_ID=$(get_record_id $DOMAIN_ID $AAAA_RECORD_NAME AAAA)
 designate record-get $DOMAIN_ID $AAAA_RECORD_ID
 
 # Verify the record is published in DNS
-verify_name_type_dns $AAAA_RECORD_NAME AAAA 2607:f0d0:1002:51::4
+ensure_record_present $AAAA_RECORD_NAME AAAA 2607:f0d0:1002:51::4
 
 # -----
 
@@ -115,7 +230,7 @@ MX_RECORD_ID=$(get_record_id $DOMAIN_ID $DOMAIN_NAME MX)
 designate record-get $DOMAIN_ID $MX_RECORD_ID
 
 # Verify the record is published in DNS
-verify_name_type_dns $DOMAIN_NAME MX "5 mail.example.com."
+ensure_record_present $DOMAIN_NAME MX "5 mail.example.com."
 
 # -----
 
@@ -128,7 +243,7 @@ SRV_RECORD_ID=$(get_record_id $DOMAIN_ID _sip._tcp.$DOMAIN_NAME SRV)
 designate record-get $DOMAIN_ID $SRV_RECORD_ID
 
 # Verify the record is published in DNS
-verify_name_type_dns _sip._tcp.$DOMAIN_NAME SRV "10 5 5060 sip.example.com."
+ensure_record_present _sip._tcp.$DOMAIN_NAME SRV "10 5 5060 sip.example.com."
 
 # -----
 
@@ -144,7 +259,7 @@ CNAME_RECORD_ID=$(get_record_id $DOMAIN_ID $CNAME_RECORD_NAME CNAME)
 designate record-get $DOMAIN_ID $CNAME_RECORD_ID
 
 # Verify the record is published in DNS
-verify_name_type_dns $CNAME_RECORD_NAME CNAME $DOMAIN_NAME
+ensure_record_present $CNAME_RECORD_NAME CNAME $DOMAIN_NAME
 
 # -----
 
@@ -166,7 +281,7 @@ designate record-list $DOMAIN_ID
 designate record-get $DOMAIN_ID $CNAME_RECORD_ID || echo "good - record was removed"
 
 # verify not in DNS anymore
-verify_name_type_dns $CNAME_RECORD_NAME CNAME $DOMAIN_NAME 1
+ensure_record_absent $CNAME_RECORD_NAME CNAME $DOMAIN_NAME
 
 # Testing Domains Delete
 # ======================
@@ -178,8 +293,8 @@ designate domain-delete $DOMAIN_ID
 designate domain-get $DOMAIN_ID || echo "good - domain was removed"
 
 # should not have SOA and NS records
-verify_name_type_dns $DOMAIN_NAME SOA $DESIGNATE_TEST_NSREC 1
-verify_name_type_dns $DOMAIN_NAME NS $DESIGNATE_TEST_NSREC 1
+ensure_record_absent $DOMAIN_NAME SOA $DESIGNATE_TEST_NSREC
+ensure_record_absent $DOMAIN_NAME NS $DESIGNATE_TEST_NSREC
 
 set +o xtrace
 echo "*********************************************************************"
diff --git a/contrib/devstack/lib/designate b/contrib/devstack/lib/designate
index bd1b14c3..17c9a69e 100644
--- a/contrib/devstack/lib/designate
+++ b/contrib/devstack/lib/designate
@@ -54,12 +54,6 @@ DESIGNATE_SERVICE_PORT_DNS=${DESIGNATE_SERVICE_PORT_DNS:-53}
 DESIGNATE_SERVICE_PORT_MDNS=${DESIGNATE_SERVICE_PORT_MDNS:-5354}
 DESIGNATE_TEST_NSREC=${DESIGNATE_TEST_NSREC:-ns1.devstack.org.}
 
-# used with dig to look up in DNS
-DESIGNATE_DIG_FLAGS="-p $DESIGNATE_SERVICE_PORT_DNS @$DESIGNATE_SERVICE_HOST"
-
-# used with dig to do an AXFR against MDNS
-DESIGNATE_DIG_AXFR_FLAGS="-p $DESIGNATE_SERVICE_PORT_MDNS @$DESIGNATE_SERVICE_HOST AXFR +tcp +nocmd"
-
 DESIGNATE_ENABLED_EXTENSIONS_V1=${DESIGNATE_ENABLED_EXTENSIONS_V1:-"quotas"}
 DESIGNATE_ENABLED_EXTENSIONS_V2=${DESIGNATE_ENABLED_EXTENSIONS_V2:-"quotas"}
 
@@ -70,99 +64,6 @@ if is_service_enabled designate && [[ -r $DESIGNATE_PLUGINS/backend-$DESIGNATE_B
     source $DESIGNATE_PLUGINS/backend-$DESIGNATE_BACKEND_DRIVER
 fi
 
-# Functions
-# ---------
-
-# lookup the given name and type in DNS
-# if it does not match the expected value, give an error
-# if $4 is given, reverse the test - assert the value is not present
-function verify_name_type_dns {
-    if [ "$DESIGNATE_BACKEND_DRIVER" = "fake" ] ; then
-        # if the backend is fake, there will be no actual DNS records
-        return 0
-    fi
-
-    # give DNS changes time to show up
-    sleep 1
-
-    # Display for debugging
-    dig $DESIGNATE_DIG_FLAGS "$1" "$2"
-    if [ -n "$4" ] ; then
-        if dig +short $DESIGNATE_DIG_FLAGS "$1" "$2" | grep "$3"; then
-            die $LINENO "Error: record $3 found in DNS, should have been removed"
-        fi
-        return 0
-    else
-        if dig +short $DESIGNATE_DIG_FLAGS "$1" "$2" | grep "$3"; then
-            return 0
-        fi
-        die $LINENO "Error: record $3 not found in DNS"
-    fi
-}
-
-# do an AXFR request to MDNS
-# if it does not match the expected value, give an error
-function verify_axfr_in_mdns {
-    # Display for debugging
-    dig $DESIGNATE_DIG_AXFR_FLAGS "$1"
-    if dig $DESIGNATE_DIG_AXFR_FLAGS "$1"; then
-        if [ -n "$2" ] ; then
-            local axfr_records=$(dig $DESIGNATE_DIG_AXFR_FLAGS "$1" | grep "$1" | wc -l)
-            if [ "$axfr_records" = "$2" ] ; then
-                return 0
-            else
-                die $LINENO "Error: AXFR to MDNS did not return the expected number of records"
-            fi
-        fi
-        return 0
-    else
-        die $LINENO "Error: AXFR to MDNS did not return a correct response"
-    fi
-}
-
-# get the domain id (uuid) given the domain name
-# if REQUIRED is set, die with an error if name not found
-function get_domain_id {
-    local domain_name=$1
-    local required=$2
-    local domain_id=$(designate domain-list | egrep " $domain_name " | get_field 1)
-    if [ "$required" = "1" ] ; then
-        die_if_not_set $LINENO domain_id "Failure retrieving DOMAIN_ID"
-    fi
-    echo "$domain_id"
-}
-
-
-# get the domain_name given the id
-function get_domain_name {
-    designate domain-list | grep "$1" | get_field 2
-}
-
-# if the given domain does not exist, it will be created
-# the domain_id of the domain will be returned
-function get_or_create_domain_id {
-    local domainid=$(get_domain_id "$1")
-    if [[ -z "$domainid" ]]; then
-        designate domain-create --name $1 --email admin@devstack.org --ttl 86400 --description "domain $1" 1>&2
-        domainid=$(designate domain-list | grep "$1" | get_field 1)
-    fi
-    echo $domainid
-}
-
-# get the record id (uuid) given the record name and domain id
-# if REQUIRED is set, die with an error if name not found
-function get_record_id {
-    local domain_id=$1
-    local record_name=$2
-    local record_type=$3
-    local required=$4
-    local record_id=$(designate record-list $domain_id | egrep " $record_name " | egrep " $record_type " | get_field 1)
-    if [ "$required" = "1" ] ; then
-        die_if_not_set $LINENO record_id "Failure retrieving RECORD_ID"
-    fi
-    echo "$record_id"
-}
-
 # cleanup_designate - Remove residual data files, anything left over from previous
 # runs that a clean run would need to clean up
 function cleanup_designate {