Browse Source

Fix network connectivity issues

Master node needs an outbound internet connection to download repository
metadata during admin node bootstrap process. We need to wait until it
starts building docker containers, then enable the connection and wait
until the bootstrap process completes.

Partial-Bug: 1525967
Change-Id: I0ca296cb5f727d2b6a5708355956c30e4ccf283b
Dmitry Bilunov 3 years ago
parent
commit
e61354ccd2
2 changed files with 34 additions and 25 deletions
  1. 5
    2
      actions/master-node-create-and-install.sh
  2. 29
    23
      functions/product.sh

+ 5
- 2
actions/master-node-create-and-install.sh View File

@@ -57,12 +57,15 @@ fi
57 57
 echo
58 58
 start_vm $name
59 59
 
60
-# Wait until the machine gets installed and Puppet completes its run
61
-wait_for_product_vm_to_install $vm_master_ip $vm_master_username $vm_master_password "$vm_master_prompt"
60
+# Wait until product VM needs outbound network/internet access
61
+wait_for_product_vm_to_download $vm_master_ip $vm_master_username $vm_master_password "$vm_master_prompt"
62 62
 
63 63
 # Enable outbound network/internet access for the machine
64 64
 enable_outbound_network_for_product_vm $vm_master_ip $vm_master_username $vm_master_password "$vm_master_prompt" 3 $vm_master_nat_gateway
65 65
 
66
+# Wait until the machine gets installed and Puppet completes its run
67
+wait_for_product_vm_to_install $vm_master_ip $vm_master_username $vm_master_password "$vm_master_prompt"
68
+
66 69
 # Report success
67 70
 echo
68 71
 echo "Master node has been installed."

+ 29
- 23
functions/product.sh View File

@@ -22,11 +22,13 @@ source ./functions/shell.sh
22 22
 
23 23
 ssh_options='-oConnectTimeout=5 -oStrictHostKeyChecking=no -oCheckHostIP=no -oUserKnownHostsFile=/dev/null -oRSAAuthentication=no -oPubkeyAuthentication=no'
24 24
 
25
-is_product_vm_operational() {
25
+wait_for_line_in_puppet_bootstrap() {
26 26
     ip=$1
27 27
     username=$2
28 28
     password=$3
29 29
     prompt=$4
30
+    goodline=$5
31
+    badline=$6
30 32
 
31 33
     # Log in into the VM, see if Puppet has completed its run
32 34
     # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line
@@ -37,36 +39,40 @@ is_product_vm_operational() {
37 39
         expect "*?assword:*"
38 40
         send "$password\r"
39 41
         expect "$prompt"
40
-        send "grep --color=none 'Fuel node deployment' /var/log/puppet/bootstrap_admin_node.log\r"
42
+        send "egrep --color=none -e '$goodline' -e '$badline' /var/log/puppet/bootstrap_admin_node.log\r"
41 43
         expect "$prompt"
42 44
         send "logout\r"
43 45
         expect "$prompt"
44 46
 ENDOFEXPECT
45 47
     )
46 48
 
47
-    # When you are launching command in a sub-shell, there are issues with IFS (internal field separator)
48
-    # and parsing output as a set of strings. So, we are saving original IFS, replacing it, iterating over lines,
49
-    # and changing it back to normal
50
-    #
51
-    # http://blog.edwards-research.com/2010/01/quick-bash-trick-looping-through-output-lines/
52
-    OIFS="${IFS}"
53
-    NIFS=$'\n'
54
-    IFS="${NIFS}"
49
+    echo "$result" | egrep "$badline" >&2 && return 1
50
+    echo "$result" | egrep -q "$goodline" && return 0
51
+    return 1
52
+}
55 53
 
56
-    for line in $result; do
57
-        IFS="${OIFS}"
58
-        if [[ "$line" == Fuel*complete* ]]; then
59
-            IFS="${NIFS}"
60
-            return 0;
61
-        elif [[ "$line" == Fuel*FAILED* ]]; then
62
-            IFS="${NIFS}"
63
-            echo "$line" 1>&2
64
-            exit 1
54
+is_product_vm_operational() {
55
+    wait_for_line_in_puppet_bootstrap "$@" "^Fuel.*complete" "^Fuel.*FAILED"
56
+}
57
+
58
+wait_for_product_vm_to_download() {
59
+    ip=$1
60
+    username=$2
61
+    password=$3
62
+    prompt=$4
63
+
64
+    echo "Waiting for product VM to download files. Please do NOT abort the script..."
65
+
66
+    # Loop until master node gets successfully installed
67
+    maxdelay=3000
68
+    while ! wait_for_line_in_puppet_bootstrap $ip $username $password "$prompt" "dockerctl build all|^Fuel.*complete" "^Fuel.*FAILED"; do
69
+        sleep 5
70
+        ((waited += 5))
71
+        if (( waited >= maxdelay )); then
72
+          echo "Installation timed out! ($maxdelay seconds)" 1>&2
73
+          exit 1
65 74
         fi
66
-        IFS="${NIFS}"
67 75
     done
68
-
69
-    return 1
70 76
 }
71 77
 
72 78
 wait_for_product_vm_to_install() {
@@ -153,7 +159,7 @@ enable_outbound_network_for_product_vm() {
153 159
     if execute test -f /etc/resolv.conf ; then
154 160
       nameserver="$(execute grep '^nameserver' /etc/resolv.conf | grep -v 'nameserver\s\s*127.' | head -3)"
155 161
     fi
156
-    if [ -z "$nameserver" -a execute test -x /usr/bin/nmcli ]; then
162
+    if [ -z "$nameserver" ] && execute test -x /usr/bin/nmcli; then
157 163
       # Get DNS from network manager
158 164
       if [ -n "`execute LANG=C nmcli nm | grep \"running\s\+connected\"`" ]; then
159 165
         nameserver="$(execute nmcli dev list | grep 'IP[46].DNS' | sed -e 's/IP[46]\.DNS\[[0-9]\+\]:\s\+/nameserver /'| grep -v 'nameserver\s\s*127.' | head -3)"

Loading…
Cancel
Save