Browse Source

Liveness/Readiness probe for Neutron server and its agents

Health_probe for neutron pods accomplish both liveness and
readiness probe.

Neutron DHCP/L3/OVS agents:
Sends an RPC call with a non-existence method to agent’s queue.
Assumes no other agent subscribed to tunnel-update queue other
than OVS. Probe is success if agent returns with NoSuchMethod
error.

Neutron Metadata agent:
Sends a message to Unix Domain Socket opened by Metadata agent.
Probe is success if agent returns with HTTP status 404.

In both the cases, if agent is not reachable or fails to
respond in time, returns failure to probe.

Readiness probe for Neutron L3/DHCP/Metadata/SRIOV agents
Following are the operations executed on the pod as part of
readiness probe on the neutron agents:
- Check if the agent process is up and running.
- Retrieve the sockets associated with the process from the /proc fs.
- Check the status of tcp sockets related to Rabbitmq communication.
- Check the reachability of the rabbitmq message bus from the agent.
- For SRIOV Agent, check if VFs are configured properly for the
configured NICs in sriov_agent.ini conf file

Change-Id: Ib99ceaabbad1d1e0faf34cc74314da9aa688fa0a
Hemachandra Reddy 3 months ago
parent
commit
da508727b6

+ 266
- 0
neutron/templates/bin/_health-probe.py.tpl View File

@@ -0,0 +1,266 @@
1
+#!/usr/bin/env python2
2
+
3
+# Copyright 2019 The Openstack-Helm Authors.
4
+#
5
+# Licensed under the Apache License, Version 2.0 (the "License");
6
+# you may not use this file except in compliance with the License.
7
+# You may obtain a copy of the License at
8
+#
9
+#     http://www.apache.org/licenses/LICENSE-2.0
10
+#
11
+# Unless required by applicable law or agreed to in writing, software
12
+# distributed under the License is distributed on an "AS IS" BASIS,
13
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+# See the License for the specific language governing permissions and
15
+# limitations under the License.
16
+
17
+"""
18
+Health probe script for OpenStack agents that uses RPC/unix domain socket for
19
+communication. Sends message to agent through rpc call method and expects a
20
+reply. It is expected to receive a failure from the agent's RPC server as the
21
+method does not exist.
22
+
23
+Script returns failure to Kubernetes only when
24
+  a. agent is not reachable or
25
+  b. agent times out sending a reply.
26
+
27
+sys.stderr.write() writes to pod's events on failures.
28
+
29
+Usage example for Neutron L3 agent:
30
+# python health-probe.py --config-file /etc/neutron/neutron.conf \
31
+#  --config-file /etc/neutron/l3_agent.ini --agent-queue-name l3_agent
32
+
33
+Usage example for Neutron metadata agent:
34
+# python health-probe.py --config-file /etc/neutron/neutron.conf \
35
+#  --config-file /etc/neutron/metadata_agent.ini
36
+"""
37
+
38
+import httplib2
39
+from six.moves import http_client as httplib
40
+import os
41
+import psutil
42
+import socket
43
+import sys
44
+
45
+from oslo_config import cfg
46
+from oslo_context import context
47
+from oslo_log import log
48
+import oslo_messaging
49
+
50
+rabbit_port = 5672
51
+tcp_established = "ESTABLISHED"
52
+log.logging.basicConfig(level=log.ERROR)
53
+
54
+
55
+def check_agent_status(transport):
56
+    """Verify agent status. Return success if agent consumes message"""
57
+    try:
58
+        target = oslo_messaging.Target(topic=cfg.CONF.agent_queue_name,
59
+                                       server=socket.gethostname())
60
+        client = oslo_messaging.RPCClient(transport, target,
61
+                                          timeout=60,
62
+                                          retry=2)
63
+        client.call(context.RequestContext(),
64
+                    'pod_health_probe_method_ignore_errors')
65
+    except oslo_messaging.exceptions.MessageDeliveryFailure:
66
+        # Log to pod events
67
+        sys.stderr.write("Health probe unable to reach message bus")
68
+        sys.exit(0)  # return success
69
+    except oslo_messaging.rpc.client.RemoteError as re:
70
+        if ("Endpoint does not support RPC method" in re.message) or \
71
+                ("Endpoint does not support RPC version" in re.message):
72
+            sys.exit(0)  # Call reached the agent
73
+        else:
74
+            sys.stderr.write("Health probe unable to reach agent")
75
+            sys.exit(1)  # return failure
76
+    except oslo_messaging.exceptions.MessagingTimeout:
77
+        sys.stderr.write("Health probe timed out. Agent is down or response "
78
+                         "timed out")
79
+        sys.exit(1)  # return failure
80
+    except Exception as ex:
81
+        sys.stderr.write("Health probe caught exception sending message to "
82
+                         "agent: %s" % ex.message)
83
+        sys.exit(0)
84
+    except:
85
+        sys.stderr.write("Health probe caught exception sending message to"
86
+                         " agent")
87
+        sys.exit(0)
88
+
89
+
90
+def sriov_readiness_check():
91
+    """Checks the sriov configuration on the sriov nic's"""
92
+    return_status = 1
93
+    with open('/etc/neutron/plugins/ml2/sriov_agent.ini') as nic:
94
+        for phy in nic:
95
+            if "physical_device_mappings" in phy:
96
+                phy_dev = phy.split('=', 1)[1]
97
+                phy_dev1 = phy_dev.rstrip().split(',')
98
+                if not phy_dev1:
99
+                    sys.stderr.write("No Physical devices"
100
+                                     " configured as SRIOV NICs")
101
+                    sys.exit(1)
102
+                for intf in phy_dev1:
103
+                    phy, dev = intf.split(':')
104
+                    try:
105
+                        with open('/sys/class/net/%s/device/'
106
+                                  'sriov_numvfs' % dev) as f:
107
+                            for line in f:
108
+                                numvfs = line.rstrip('\n')
109
+                                if numvfs:
110
+                                    return_status = 0
111
+                    except IOError:
112
+                        sys.stderr.write("IOError:No sriov_numvfs config file")
113
+    sys.exit(return_status)
114
+
115
+
116
+def tcp_socket_state_check(agentq):
117
+    """Check if the tcp socket to rabbitmq is in Established state"""
118
+    rabbit_sock_count = 0
119
+    parentId = 0
120
+    if agentq == "l3_agent":
121
+        proc = "neutron-l3-agen"
122
+    elif agentq == "dhcp_agent":
123
+        proc = "neutron-dhcp-ag"
124
+    elif agentq == "q-agent-notifier-tunnel-update":
125
+        proc = "neutron-openvsw"
126
+    else:
127
+        proc = "neutron-metadat"
128
+
129
+    for pr in psutil.pids():
130
+        try:
131
+            p = psutil.Process(pr)
132
+            if p.name() == proc:
133
+                if parentId == 0:
134
+                    parentId = p.pid
135
+                else:
136
+                    if p.ppid() == parentId:
137
+                        continue
138
+                pcon = p.connections()
139
+                for con in pcon:
140
+                    try:
141
+                        port = con.raddr[1]
142
+                        status = con.status
143
+                    except IndexError:
144
+                        continue
145
+                    if port == rabbit_port and status == tcp_established:
146
+                        rabbit_sock_count = rabbit_sock_count + 1
147
+        except psutil.NoSuchProcess:
148
+            continue
149
+
150
+    if rabbit_sock_count == 0:
151
+        sys.stderr.write("RabbitMQ sockets not Established")
152
+        # Do not kill the pod if RabbitMQ is not reachable/down
153
+        if not cfg.CONF.liveness_probe:
154
+            sys.exit(1)
155
+
156
+
157
+class UnixDomainHTTPConnection(httplib.HTTPConnection):
158
+    """Connection class for HTTP over UNIX domain socket."""
159
+
160
+    def __init__(self, host, port=None, strict=None, timeout=None,
161
+                 proxy_info=None):
162
+        httplib.HTTPConnection.__init__(self, host, port, strict)
163
+        self.timeout = timeout
164
+        self.socket_path = cfg.CONF.metadata_proxy_socket
165
+
166
+    def connect(self):
167
+        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
168
+        if self.timeout:
169
+            self.sock.settimeout(self.timeout)
170
+        self.sock.connect(self.socket_path)
171
+
172
+
173
+def test_socket_liveness():
174
+    """Test if agent can respond to message over the socket"""
175
+    cfg.CONF.register_cli_opt(cfg.BoolOpt('liveness-probe', default=False,
176
+                                          required=False))
177
+    cfg.CONF(sys.argv[1:])
178
+
179
+    agentq = "metadata_agent"
180
+    tcp_socket_state_check(agentq)
181
+
182
+    try:
183
+        metadata_proxy_socket = cfg.CONF.metadata_proxy_socket
184
+    except cfg.NoSuchOptError:
185
+        cfg.CONF.register_opt(cfg.StrOpt(
186
+            'metadata_proxy_socket',
187
+            default='/var/lib/neutron/openstack-helm/metadata_proxy'))
188
+
189
+    headers = {'X-Forwarded-For': '169.254.169.254',
190
+               'X-Neutron-Router-ID': 'pod-health-probe-check-ignore-errors'}
191
+
192
+    h = httplib2.Http(timeout=30)
193
+
194
+    try:
195
+        resp, content = h.request(
196
+            'http://169.254.169.254',
197
+            method='GET',
198
+            headers=headers,
199
+            connection_type=UnixDomainHTTPConnection)
200
+    except socket.error as se:
201
+        msg = "Socket error: Health probe failed to connect to " \
202
+              "Neutron Metadata agent: "
203
+        if se.strerror:
204
+            sys.stderr.write(msg + se.strerror)
205
+        elif se.message:
206
+            sys.stderr.write(msg + se.message)
207
+        sys.exit(1)  # return failure
208
+    except Exception as ex:
209
+        sys.stderr.write("Health probe caught exception sending message to "
210
+                         "Neutron Metadata agent: %s" % ex.message)
211
+        sys.exit(0)  # return success
212
+
213
+    if resp.status >= 500:  # Probe expects HTTP error code 404
214
+        msg = "Health probe failed: Neutron Metadata agent failed to" \
215
+              " process request: "
216
+        sys.stderr.write(msg + str(resp.__dict__))
217
+        sys.exit(1)  # return failure
218
+
219
+
220
+def test_rpc_liveness():
221
+    """Test if agent can consume message from queue"""
222
+    oslo_messaging.set_transport_defaults(control_exchange='neutron')
223
+
224
+    rabbit_group = cfg.OptGroup(name='oslo_messaging_rabbit',
225
+                                title='RabbitMQ options')
226
+    cfg.CONF.register_group(rabbit_group)
227
+    cfg.CONF.register_cli_opt(cfg.StrOpt('agent-queue-name'))
228
+    cfg.CONF.register_cli_opt(cfg.BoolOpt('liveness-probe', default=False,
229
+                                          required=False))
230
+
231
+    cfg.CONF(sys.argv[1:])
232
+
233
+    try:
234
+        transport = oslo_messaging.get_transport(cfg.CONF)
235
+    except Exception as ex:
236
+        sys.stderr.write("Message bus driver load error: %s" % ex.message)
237
+        sys.exit(0)  # return success
238
+
239
+    if not cfg.CONF.transport_url or \
240
+            not cfg.CONF.agent_queue_name:
241
+        sys.stderr.write("Both message bus URL and agent queue name are "
242
+                         "required for Health probe to work")
243
+        sys.exit(0)  # return success
244
+
245
+    try:
246
+        cfg.CONF.set_override('rabbit_max_retries', 2,
247
+                              group=rabbit_group)  # 3 attempts
248
+    except cfg.NoSuchOptError as ex:
249
+        cfg.CONF.register_opt(cfg.IntOpt('rabbit_max_retries', default=2),
250
+                              group=rabbit_group)
251
+
252
+    agentq = cfg.CONF.agent_queue_name
253
+    tcp_socket_state_check(agentq)
254
+
255
+    check_agent_status(transport)
256
+
257
+
258
+if __name__ == "__main__":
259
+    if "sriov_agent.ini" in ','.join(sys.argv):
260
+        sriov_readiness_check()
261
+    elif "metadata_agent.ini" not in ','.join(sys.argv):
262
+        test_rpc_liveness()
263
+    else:
264
+        test_socket_liveness()
265
+
266
+    sys.exit(0)  # return success

+ 2
- 0
neutron/templates/configmap-bin.yaml View File

@@ -45,6 +45,8 @@ data:
45 45
 {{- include "helm-toolkit.scripts.keystone_endpoints" . | indent 4 }}
46 46
   ks-user.sh: |
47 47
 {{- include "helm-toolkit.scripts.keystone_user" . | indent 4 }}
48
+  health-probe.py: |
49
+{{ tuple "bin/_health-probe.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
48 50
   neutron-dhcp-agent.sh: |
49 51
 {{ tuple "bin/_neutron-dhcp-agent.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
50 52
   neutron-l3-agent.sh: |

+ 33
- 0
neutron/templates/daemonset-dhcp-agent.yaml View File

@@ -65,6 +65,35 @@ spec:
65 65
 {{ tuple $envAll $envAll.Values.pod.resources.agent.dhcp | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
66 66
           securityContext:
67 67
             privileged: true
68
+          readinessProbe:
69
+            exec:
70
+              command:
71
+              - python
72
+              - /tmp/health-probe.py
73
+              - --config-file
74
+              - /etc/neutron/neutron.conf
75
+              - --config-file
76
+              - /etc/neutron/dhcp_agent.ini
77
+              - --agent-queue-name
78
+              - dhcp_agent
79
+            initialDelaySeconds: 30
80
+            periodSeconds: 15
81
+            timeoutSeconds: 65
82
+          livenessProbe:
83
+            exec:
84
+              command:
85
+              - python
86
+              - /tmp/health-probe.py
87
+              - --config-file
88
+              - /etc/neutron/neutron.conf
89
+              - --config-file
90
+              - /etc/neutron/dhcp_agent.ini
91
+              - --agent-queue-name
92
+              - dhcp_agent
93
+              - --liveness-probe
94
+            initialDelaySeconds: 120
95
+            periodSeconds: 90
96
+            timeoutSeconds: 70
68 97
           command:
69 98
             - /tmp/neutron-dhcp-agent.sh
70 99
           volumeMounts:
@@ -72,6 +101,10 @@ spec:
72 101
               mountPath: /tmp/neutron-dhcp-agent.sh
73 102
               subPath: neutron-dhcp-agent.sh
74 103
               readOnly: true
104
+            - name: neutron-bin
105
+              mountPath: /tmp/health-probe.py
106
+              subPath: health-probe.py
107
+              readOnly: true
75 108
             - name: neutron-etc
76 109
               mountPath: /etc/neutron/neutron.conf
77 110
               subPath: neutron.conf

+ 33
- 0
neutron/templates/daemonset-l3-agent.yaml View File

@@ -65,6 +65,35 @@ spec:
65 65
 {{ tuple $envAll $envAll.Values.pod.resources.agent.l3 | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
66 66
           securityContext:
67 67
             privileged: true
68
+          readinessProbe:
69
+            exec:
70
+              command:
71
+              - python
72
+              - /tmp/health-probe.py
73
+              - --config-file
74
+              - /etc/neutron/neutron.conf
75
+              - --config-file
76
+              - /etc/neutron/l3_agent.ini
77
+              - --agent-queue-name
78
+              - l3_agent
79
+            initialDelaySeconds: 30
80
+            periodSeconds: 15
81
+            timeoutSeconds: 65
82
+          livenessProbe:
83
+            exec:
84
+              command:
85
+              - python
86
+              - /tmp/health-probe.py
87
+              - --config-file
88
+              - /etc/neutron/neutron.conf
89
+              - --config-file
90
+              - /etc/neutron/l3_agent.ini
91
+              - --agent-queue-name
92
+              - l3_agent
93
+              - --liveness-probe
94
+            initialDelaySeconds: 120
95
+            periodSeconds: 90
96
+            timeoutSeconds: 70
68 97
           command:
69 98
             - /tmp/neutron-l3-agent.sh
70 99
           volumeMounts:
@@ -72,6 +101,10 @@ spec:
72 101
               mountPath: /tmp/neutron-l3-agent.sh
73 102
               subPath: neutron-l3-agent.sh
74 103
               readOnly: true
104
+            - name: neutron-bin
105
+              mountPath: /tmp/health-probe.py
106
+              subPath: health-probe.py
107
+              readOnly: true
75 108
             - name: neutron-etc
76 109
               mountPath: /etc/neutron/neutron.conf
77 110
               subPath: neutron.conf

+ 29
- 0
neutron/templates/daemonset-metadata-agent.yaml View File

@@ -86,6 +86,31 @@ spec:
86 86
 {{ tuple $envAll $envAll.Values.pod.resources.agent.metadata | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
87 87
           securityContext:
88 88
             privileged: true
89
+          readinessProbe:
90
+            exec:
91
+              command:
92
+              - python
93
+              - /tmp/health-probe.py
94
+              - --config-file
95
+              - /etc/neutron/neutron.conf
96
+              - --config-file
97
+              - /etc/neutron/metadata_agent.ini
98
+            initialDelaySeconds: 30
99
+            periodSeconds: 15
100
+            timeoutSeconds: 35
101
+          livenessProbe:
102
+            exec:
103
+              command:
104
+              - python
105
+              - /tmp/health-probe.py
106
+              - --config-file
107
+              - /etc/neutron/neutron.conf
108
+              - --config-file
109
+              - /etc/neutron/metadata_agent.ini
110
+              - --liveness-probe
111
+            initialDelaySeconds: 90
112
+            periodSeconds: 60
113
+            timeoutSeconds: 45
89 114
           command:
90 115
             - /tmp/neutron-metadata-agent.sh
91 116
           volumeMounts:
@@ -93,6 +118,10 @@ spec:
93 118
               mountPath: /tmp/neutron-metadata-agent.sh
94 119
               subPath: neutron-metadata-agent.sh
95 120
               readOnly: true
121
+            - name: neutron-bin
122
+              mountPath: /tmp/health-probe.py
123
+              subPath: health-probe.py
124
+              readOnly: true
96 125
             - name: neutron-etc
97 126
               mountPath: /etc/neutron/neutron.conf
98 127
               subPath: neutron.conf

+ 19
- 0
neutron/templates/daemonset-ovs-agent.yaml View File

@@ -157,11 +157,30 @@ spec:
157 157
                 - bash
158 158
                 - -c
159 159
                 - 'ovs-vsctl list-br | grep -q br-int'
160
+          livenessProbe:
161
+            exec:
162
+              command:
163
+              - python
164
+              - /tmp/health-probe.py
165
+              - --config-file
166
+              - /etc/neutron/neutron.conf
167
+              - --config-file
168
+              - /etc/neutron/plugins/ml2/openvswitch_agent.ini
169
+              - --agent-queue-name
170
+              - q-agent-notifier-tunnel-update
171
+              - --liveness-probe
172
+            initialDelaySeconds: 120
173
+            periodSeconds: 90
174
+            timeoutSeconds: 70
160 175
           volumeMounts:
161 176
             - name: neutron-bin
162 177
               mountPath: /tmp/neutron-openvswitch-agent.sh
163 178
               subPath: neutron-openvswitch-agent.sh
164 179
               readOnly: true
180
+            - name: neutron-bin
181
+              mountPath: /tmp/health-probe.py
182
+              subPath: health-probe.py
183
+              readOnly: true
165 184
             - name: pod-shared
166 185
               mountPath: /tmp/pod-shared
167 186
             - name: neutron-etc

+ 16
- 0
neutron/templates/daemonset-sriov-agent.yaml View File

@@ -128,11 +128,27 @@ spec:
128 128
             privileged: true
129 129
           command:
130 130
             - /tmp/neutron-sriov-agent.sh
131
+          readinessProbe:
132
+            exec:
133
+              command:
134
+              - python
135
+              - /tmp/health-probe.py
136
+              - --config-file
137
+              - /etc/neutron/neutron.conf
138
+              - --config-file
139
+              - /etc/neutron/sriov_agent.ini
140
+            initialDelaySeconds: 30
141
+            periodSeconds: 15
142
+            timeoutSeconds: 10
131 143
           volumeMounts:
132 144
             - name: neutron-bin
133 145
               mountPath: /tmp/neutron-sriov-agent.sh
134 146
               subPath: neutron-sriov-agent.sh
135 147
               readOnly: true
148
+            - name: neutron-bin
149
+              mountPath: /tmp/health-probe.py
150
+              subPath: health-probe.py
151
+              readOnly: true
136 152
             - name: pod-shared
137 153
               mountPath: /tmp/pod-shared
138 154
             - name: neutron-etc

+ 4
- 0
neutron/templates/deployment-server.yaml View File

@@ -78,6 +78,10 @@ spec:
78 78
           readinessProbe:
79 79
             tcpSocket:
80 80
               port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
81
+          livenessProbe:
82
+            tcpSocket:
83
+              port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
84
+            initialDelaySeconds: 60
81 85
           volumeMounts:
82 86
             - name: neutron-bin
83 87
               mountPath: /tmp/neutron-server.sh

Loading…
Cancel
Save