Browse Source

(fix) Make rackd stateful

- Use a statefulset and PVC to make rackd systemid assignment
  stateful between pod restarts. This is to alleviate instability
  in MAAS upgrades.

Change-Id: Iea5c3d3897b561d4ba479203ee6aec5885282e1a
Scott Hussey 10 months ago
parent
commit
b09fee26b8

+ 6
- 0
charts/maas/templates/bin/_register-rack-controller.sh.tpl View File

@@ -2,6 +2,12 @@
2 2
 
3 3
 set -x
4 4
 
5
+if [[ -r ~maas/maas_id && -r ~maas/secret ]]
6
+then
7
+  echo "Found existing maas_id and secret, assuming already registered."
8
+  exit 0
9
+fi
10
+
5 11
 echo "register-rack-controller URL: ${MAAS_ENDPOINT}"
6 12
 
7 13
 # register forever

+ 8
- 3
charts/maas/templates/bin/_start.sh.tpl View File

@@ -19,14 +19,19 @@ set -ex
19 19
 # show env
20 20
 env > /tmp/env
21 21
 
22
+# Ensure PVC volumes have correct ownership
23
+
24
+chown maas:maas ~maas/
25
+chown maas:maas /etc/maas
26
+
22 27
 # MAAS must be able to ssh to libvirt hypervisors
23 28
 # to control VMs
24 29
 
25
-if [[ -d ~maas/keys ]]
30
+if [[ -r ~maas/id_rsa ]]
26 31
 then
27 32
   mkdir -p ~maas/.ssh
28
-  cp ~maas/keys/* ~maas/.ssh/
29
-  chown -R maas:maas ~maas/.ssh
33
+  cp ~maas/id_rsa ~maas/.ssh/
34
+  chown -R maas:maas ~maas/.ssh/
30 35
   chmod 700 ~maas/.ssh
31 36
   chmod 600 ~maas/.ssh/*
32 37
 fi

+ 7
- 0
charts/maas/templates/service-rack.yaml View File

@@ -0,0 +1,7 @@
1
+---
2
+apiVersion: v1
3
+kind: Service
4
+metadata:
5
+  name: maas-rack
6
+spec:
7
+  clusterIP: 'None'

charts/maas/templates/deployment-rack.yaml → charts/maas/templates/statefulset-rack.yaml View File

@@ -28,11 +28,14 @@ limitations under the License.
28 28
 {{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
29 29
 ---
30 30
 apiVersion: apps/v1beta1
31
-kind: Deployment
31
+kind: StatefulSet
32 32
 metadata:
33 33
   name: maas-rack
34 34
 spec:
35
+  serviceName: maas-rack
35 36
   replicas: {{ .Values.pod.replicas.rack }}
37
+  updateStrategy:
38
+    type: 'RollingUpdate'
36 39
   template:
37 40
     metadata:
38 41
       labels:
@@ -98,12 +101,18 @@ spec:
98 101
               mountPath: /lib/systemd/system/register-rack-controller.service
99 102
               subPath: register-rack-controller.service
100 103
               readOnly: true
104
+            - name: rackd-state
105
+              mountPath: /etc/maas
106
+              subPath: etc
107
+              readOnly: false
108
+            - name: rackd-state
109
+              mountPath: /var/lib/maas
110
+              subPath: home
111
+              readOnly: false
101 112
 {{- if .Values.manifests.secret_ssh_key }}
102
-            - name: maas-ssh
103
-              mountPath: /var/lib/maas/keys
104 113
             - name: priv-key
105 114
               subPath: PRIVATE_KEY
106
-              mountPath: /var/lib/maas/keys/id_rsa
115
+              mountPath: /var/lib/maas/id_rsa
107 116
 {{- end }}
108 117
 {{ if $mounts_maas_rack.volumeMounts }}{{ toYaml $mounts_maas_rack.volumeMounts | indent 12 }}{{ end }}
109 118
       volumes:
@@ -117,8 +126,6 @@ spec:
117 126
         - name: pod-tmp
118 127
           emptyDir: {}
119 128
 {{- if .Values.manifests.secret_ssh_key }}
120
-        - name: maas-ssh
121
-          emptyDir: {}
122 129
         - name: priv-key
123 130
           secret:
124 131
             secretName: {{ .Release.Name}}-{{ .Values.secrets.ssh_key }}
@@ -133,4 +140,14 @@ spec:
133 140
             name: maas-etc
134 141
             defaultMode: 0444
135 142
 {{ if $mounts_maas_rack.volumes }}{{ toYaml $mounts_maas_rack.volumes | indent 8 }}{{ end }}
143
+  volumeClaimTemplates:
144
+    - metadata:
145
+        name: rackd-state
146
+        annotations:
147
+          {{ .Values.storage.rackd.pvc.class_path }}: {{ .Values.storage.rackd.pvc.class_name }}
148
+      spec:
149
+        accessModes: [ "ReadWriteOnce" ]
150
+        resources:
151
+          requests:
152
+            storage: {{ .Values.storage.rackd.pvc.size }}
136 153
 {{- end }}

+ 8
- 0
charts/maas/values.yaml View File

@@ -99,6 +99,7 @@ network:
99 99
   proxy:
100 100
     node_port:
101 101
       enabled: true
102
+      # Do not change the port, hardcoded in MAAS source
102 103
       port: 31800
103 104
   gui:
104 105
     node_port:
@@ -113,6 +114,13 @@ network:
113 114
     db_service: 5432
114 115
     db_service_target: 5432
115 116
 
117
+storage:
118
+  rackd:
119
+    pvc:
120
+      class_path: volume.beta.kubernetes.io/storage-class
121
+      class_name: general
122
+      size: 5Gi
123
+
116 124
 conf:
117 125
   ssh:
118 126
     # A SSH private key strings to mount

+ 13
- 0
images/maas-rack-controller/2.3_nic_filter.patch View File

@@ -0,0 +1,13 @@
1
+diff --git a/src/provisioningserver/utils/network.py b/src/provisioningserver/utils/network.py
2
+index 48eb8fd..41d13a0 100644
3
+--- a/src/provisioningserver/utils/network.py
4
++++ b/src/provisioningserver/utils/network.py
5
+@@ -1141,7 +1141,7 @@ def get_all_interfaces_definition(annotate_with_monitored: bool=True) -> dict:
6
+     interfaces = {}
7
+     dhclient_info = get_dhclient_info()
8
+     iproute_info = get_ip_route()
9
+-    exclude_types = ["loopback", "ipip"]
10
++    exclude_types = ["loopback", "ipip", "ethernet"]
11
+     if not running_in_container():
12
+         exclude_types.append("ethernet")
13
+     ipaddr_info = {

+ 6
- 1
images/maas-rack-controller/Dockerfile View File

@@ -11,7 +11,8 @@ RUN apt-get -qq update && \
11 11
     sudo \
12 12
     software-properties-common \
13 13
     libvirt-bin \
14
-    systemd
14
+    systemd \
15
+    patch
15 16
 # Don't start any optional services except for the few we need.
16 17
 
17 18
 RUN find /etc/systemd/system \
@@ -42,6 +43,10 @@ RUN systemctl enable register-rack-controller.service
42 43
 RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump
43 44
 RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump
44 45
 
46
+# Patch so that Calico interfaces are ignored
47
+COPY 2.3_nic_filter.patch /tmp/2.3_nic_filter.patch
48
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/2.3_nic_filter.patch
49
+
45 50
 # echo journalctl logs to the container's stdout
46 51
 COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
47 52
 RUN mkdir -p /etc/systemd/system/basic.target.wants ;\

Loading…
Cancel
Save