Browse Source

Use Kubernetes lease endpoint reconciler

In the resiliency gate:
* Enable the --endpoint-reconciler-type=least option for the apiserver.
* Extract etcd validation into its own stages.
* Test joining a node while one control plane node is down.

Change-Id: Id89b0816e91ab6427c5e2f4833ad4ec4e1e3d133
Depends-On: I2150d40e917567a4072a1565c1b96089f3d6fd2b
Mark Burnett 10 months ago
parent
commit
15658a088e

+ 7
- 0
examples/basic/Genesis.yaml View File

@@ -9,6 +9,13 @@ metadata:
9 9
 data:
10 10
   hostname: n0
11 11
   ip: 192.168.77.10
12
+  apiserver:
13
+    command_prefix:
14
+      - /apiserver
15
+      - --authorization-mode=Node,RBAC
16
+      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
17
+      - --service-cluster-ip-range=10.96.0.0/16
18
+      - --endpoint-reconciler-type=lease
12 19
   armada:
13 20
     target_manifest: cluster-bootstrap
14 21
   labels:

+ 6
- 0
examples/basic/armada-resources.yaml View File

@@ -716,6 +716,12 @@ data:
716 716
   upgrade:
717 717
     no_hooks: true
718 718
   values:
719
+    command_prefix:
720
+      - /apiserver
721
+      - --authorization-mode=Node,RBAC
722
+      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
723
+      - --service-cluster-ip-range=10.96.0.0/16
724
+      - --endpoint-reconciler-type=lease
719 725
     apiserver:
720 726
       etcd:
721 727
         endpoints: https://127.0.0.1:2378

+ 1
- 1
tools/g2/lib/validate.sh View File

@@ -14,7 +14,7 @@ validate_etcd_membership() {
14 14
 
15 15
     # NOTE(mark-burnett): Wait a moment for disks in test environment to settle.
16 16
     sleep 10
17
-    log Validating "${CLUSTER}" etcd membership via "${VM}"
17
+    log Validating "${CLUSTER}" etcd membership via "${VM}" for members: "${EXPECTED_MEMBERS[@]}"
18 18
     FOUND_MEMBERS=$(etcdctl_member_list "${CLUSTER}" "${VM}" | tr '\n' ' ' | sed 's/ $//')
19 19
 
20 20
     if [[ "x${EXPECTED_MEMBERS}" != "x${FOUND_MEMBERS}" ]]; then

+ 22
- 0
tools/g2/manifests/one.json View File

@@ -0,0 +1,22 @@
1
+{
2
+  "configuration": [
3
+    "promenade/schemas"
4
+  ],
5
+  "stages": [
6
+    {
7
+      "name": "Gate Setup",
8
+      "script": "gate-setup.sh"
9
+    },
10
+    {
11
+      "name": "Create VMs",
12
+      "script": "create-vms.sh"
13
+    }
14
+  ],
15
+  "vm": {
16
+    "memory": 20480,
17
+    "names": [
18
+      "n0"
19
+    ],
20
+    "vcpus": 4
21
+  }
22
+}

+ 45
- 4
tools/g2/manifests/resiliency.json View File

@@ -48,7 +48,14 @@
48 48
         "-l", "kubernetes-controller-manager=enabled",
49 49
         "-l", "kubernetes-etcd=enabled",
50 50
         "-l", "kubernetes-scheduler=enabled",
51
-        "-l", "ucp-control-plane=enabled",
51
+        "-l", "ucp-control-plane=enabled"
52
+      ]
53
+    },
54
+    {
55
+      "name": "Check initial etcd cluster",
56
+      "script": "check-etcd-health.sh",
57
+      "arguments": [
58
+        "-w", "10",
52 59
         "-e", "kubernetes n0 n0 n1 n2",
53 60
         "-e", "calico n0 n0 n1 n2"
54 61
       ]
@@ -61,6 +68,14 @@
61 68
         "-n", "n3"
62 69
       ]
63 70
     },
71
+    {
72
+      "name": "Power off n2",
73
+      "script": "power-down-node.sh",
74
+      "arguments": [
75
+        "-s",
76
+        "-n", "n2"
77
+      ]
78
+    },
64 79
     {
65 80
       "name": "Update Generated Certs",
66 81
       "script": "generate-certificates.sh",
@@ -80,7 +95,21 @@
80 95
         "-l", "kubernetes-controller-manager=enabled",
81 96
         "-l", "kubernetes-etcd=enabled",
82 97
         "-l", "kubernetes-scheduler=enabled",
83
-        "-l", "ucp-control-plane=enabled",
98
+        "-l", "ucp-control-plane=enabled"
99
+      ]
100
+    },
101
+    {
102
+      "name": "Power up n2",
103
+      "script": "power-up-node.sh",
104
+      "arguments": [
105
+        "-n", "n2"
106
+      ]
107
+    },
108
+    {
109
+      "name": "Check full etcd cluster",
110
+      "script": "check-etcd-health.sh",
111
+      "arguments": [
112
+        "-w", "30",
84 113
         "-e", "kubernetes n0 n0 n1 n2 n3",
85 114
         "-e", "calico n0 n0 n1 n2 n3"
86 115
       ]
@@ -91,7 +120,13 @@
91 120
       "arguments": [
92 121
         "-v", "n1",
93 122
         "-n", "n0",
94
-        "-r",
123
+        "-r"
124
+      ]
125
+    },
126
+    {
127
+      "name": "Check post-teardown etcd cluster",
128
+      "script": "check-etcd-health.sh",
129
+      "arguments": [
95 130
         "-e", "kubernetes n1 n1 n2 n3",
96 131
         "-e", "calico n1 n1 n2 n3"
97 132
       ]
@@ -102,7 +137,13 @@
102 137
       "arguments": [
103 138
         "-v", "n1",
104 139
         "-n", "n0",
105
-        "-l", "ucp-control-plane=enabled",
140
+        "-l", "ucp-control-plane=enabled"
141
+      ]
142
+    },
143
+    {
144
+      "name": "Check final etcd cluster",
145
+      "script": "check-etcd-health.sh",
146
+      "arguments": [
106 147
         "-e", "kubernetes n1 n1 n2 n3",
107 148
         "-e", "calico n1 n1 n2 n3"
108 149
       ]

+ 38
- 0
tools/g2/stages/check-etcd-health.sh View File

@@ -0,0 +1,38 @@
1
+#!/usr/bin/env bash
2
+
3
+set -eu
4
+
5
+source "${GATE_UTILS}"
6
+
7
+declare -a ETCD_CLUSTERS
8
+
9
+WAIT_BEFORE_CHECK=0
10
+
11
+while getopts "e:w:" opt; do
12
+    case "${opt}" in
13
+        e)
14
+            ETCD_CLUSTERS+=("${OPTARG}")
15
+            ;;
16
+        w)
17
+            WAIT_BEFORE_CHECK="${OPTARG}"
18
+            ;;
19
+        *)
20
+            echo "Unknown option"
21
+            exit 1
22
+            ;;
23
+    esac
24
+done
25
+shift $((OPTIND-1))
26
+
27
+if [ $# -gt 0 ]; then
28
+    echo "Unknown arguments specified: ${*}"
29
+    exit 1
30
+fi
31
+
32
+log Waiting "${WAIT_BEFORE_CHECK}" seconds before checking cluster health.
33
+sleep "${WAIT_BEFORE_CHECK}"
34
+
35
+for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
36
+    IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
37
+    validate_etcd_membership "${etcd_validation_args[@]}"
38
+done

+ 1
- 13
tools/g2/stages/join-nodes.sh View File

@@ -4,7 +4,6 @@ set -eu
4 4
 
5 5
 source "${GATE_UTILS}"
6 6
 
7
-declare -a ETCD_CLUSTERS
8 7
 declare -a LABELS
9 8
 declare -a NODES
10 9
 
@@ -12,11 +11,8 @@ GET_KEYSTONE_TOKEN=0
12 11
 USE_DECKHAND=0
13 12
 DECKHAND_REVISION=''
14 13
 
15
-while getopts "d:e:l:n:tv:" opt; do
14
+while getopts "d:l:n:tv:" opt; do
16 15
     case "${opt}" in
17
-        e)
18
-            ETCD_CLUSTERS+=("${OPTARG}")
19
-            ;;
20 16
         d)
21 17
             USE_DECKHAND=1
22 18
             DECKHAND_REVISION=${OPTARG}
@@ -48,7 +44,6 @@ fi
48 44
 
49 45
 SCRIPT_DIR="${TEMP_DIR}/curled-scripts"
50 46
 
51
-echo Etcd Clusters: "${ETCD_CLUSTERS[@]}"
52 47
 echo Labels: "${LABELS[@]}"
53 48
 echo Nodes: "${NODES[@]}"
54 49
 
@@ -86,10 +81,3 @@ for NAME in "${NODES[@]}"; do
86 81
     rsync_cmd "${SCRIPT_DIR}/join-${NAME}.sh" "${NAME}:/root/promenade/"
87 82
     ssh_cmd "${NAME}" "/root/promenade/join-${NAME}.sh" 2>&1 | tee -a "${LOG_FILE}"
88 83
 done
89
-
90
-sleep 10
91
-
92
-for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
93
-    IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
94
-    validate_etcd_membership "${etcd_validation_args[@]}"
95
-done

+ 31
- 0
tools/g2/stages/power-down-node.sh View File

@@ -0,0 +1,31 @@
1
+#!/usr/bin/env bash
2
+
3
+set -eu
4
+
5
+source "${GATE_UTILS}"
6
+
7
+declare -a NODES
8
+SYNC_BEFORE_STOP=0
9
+
10
+while getopts "n:s" opt; do
11
+    case "${opt}" in
12
+        n)
13
+            NODES+=("${OPTARG}")
14
+            ;;
15
+        s)
16
+            SYNC_BEFORE_STOP=1
17
+            ;;
18
+        *)
19
+            echo "Unknown option"
20
+            exit 1
21
+            ;;
22
+    esac
23
+done
24
+shift $((OPTIND-1))
25
+
26
+for node in "${NODES[@]}"; do
27
+    if [[ $SYNC_BEFORE_STOP == 1 ]]; then
28
+        ssh_cmd "${node}" sync
29
+    fi
30
+    vm_stop "${node}"
31
+done

+ 24
- 0
tools/g2/stages/power-up-node.sh View File

@@ -0,0 +1,24 @@
1
+#!/usr/bin/env bash
2
+
3
+set -eu
4
+
5
+source "${GATE_UTILS}"
6
+
7
+declare -a NODES
8
+
9
+while getopts "n:s" opt; do
10
+    case "${opt}" in
11
+        n)
12
+            NODES+=("${OPTARG}")
13
+            ;;
14
+        *)
15
+            echo "Unknown option"
16
+            exit 1
17
+            ;;
18
+    esac
19
+done
20
+shift $((OPTIND-1))
21
+
22
+for node in "${NODES[@]}"; do
23
+    vm_start "${node}"
24
+done

+ 1
- 10
tools/g2/stages/teardown-nodes.sh View File

@@ -4,16 +4,12 @@ set -eu
4 4
 
5 5
 source "${GATE_UTILS}"
6 6
 
7
-declare -a ETCD_CLUSTERS
8 7
 declare -a NODES
9 8
 
10 9
 RECREATE=0
11 10
 
12
-while getopts "e:n:rv:" opt; do
11
+while getopts "n:rv:" opt; do
13 12
     case "${opt}" in
14
-        e)
15
-            ETCD_CLUSTERS+=("${OPTARG}")
16
-            ;;
17 13
         n)
18 14
             NODES+=("${OPTARG}")
19 15
             ;;
@@ -44,8 +40,3 @@ for NAME in "${NODES[@]}"; do
44 40
         vm_create "${NAME}"
45 41
     fi
46 42
 done
47
-
48
-for etcd_validation_string in "${ETCD_CLUSTERS[@]}"; do
49
-    IFS=' ' read -a etcd_validation_args <<<"${etcd_validation_string}"
50
-    validate_etcd_membership "${etcd_validation_args[@]}"
51
-done

Loading…
Cancel
Save