(fix) Make rackd stateful

- Use a statefulset and PVC to make rackd systemid assignment
  stateful between pod restarts. This is to alleviate instability
  in MAAS upgrades.

Change-Id: Iea5c3d3897b561d4ba479203ee6aec5885282e1a

charts/maas/templates/bin/_register-rack-controller.sh.tpl

@@ -2,6 +2,12 @@
 
 set -x
 
+if [[ -r ~maas/maas_id && -r ~maas/secret ]]
+then
+  echo "Found existing maas_id and secret, assuming already registered."
+  exit 0
+fi
+
 echo "register-rack-controller URL: ${MAAS_ENDPOINT}"
 
 # register forever

charts/maas/templates/bin/_start.sh.tpl

@@ -19,14 +19,19 @@ set -ex
 # show env
 env > /tmp/env
 
+# Ensure PVC volumes have correct ownership
+
+chown maas:maas ~maas/
+chown maas:maas /etc/maas
+
 # MAAS must be able to ssh to libvirt hypervisors
 # to control VMs
 
-if [[ -d ~maas/keys ]]
+if [[ -r ~maas/id_rsa ]]
 then
   mkdir -p ~maas/.ssh
-  cp ~maas/keys/* ~maas/.ssh/
+  cp ~maas/id_rsa ~maas/.ssh/
-  chown -R maas:maas ~maas/.ssh
+  chown -R maas:maas ~maas/.ssh/
   chmod 700 ~maas/.ssh
   chmod 600 ~maas/.ssh/*
 fi

charts/maas/templates/service-rack.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: maas-rack
+spec:
+  clusterIP: 'None'

charts/maas/templates/statefulset-rack.yaml

@@ -28,11 +28,14 @@ limitations under the License.
 {{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
 ---
 apiVersion: apps/v1beta1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: maas-rack
 spec:
+  serviceName: maas-rack
   replicas: {{ .Values.pod.replicas.rack }}
+  updateStrategy:
+    type: 'RollingUpdate'
   template:
     metadata:
       labels:
@@ -98,12 +101,18 @@ spec:
               mountPath: /lib/systemd/system/register-rack-controller.service
               subPath: register-rack-controller.service
               readOnly: true
+            - name: rackd-state
+              mountPath: /etc/maas
+              subPath: etc
+              readOnly: false
+            - name: rackd-state
+              mountPath: /var/lib/maas
+              subPath: home
+              readOnly: false
 {{- if .Values.manifests.secret_ssh_key }}
-            - name: maas-ssh
-              mountPath: /var/lib/maas/keys
             - name: priv-key
               subPath: PRIVATE_KEY
-              mountPath: /var/lib/maas/keys/id_rsa
+              mountPath: /var/lib/maas/id_rsa
 {{- end }}
 {{ if $mounts_maas_rack.volumeMounts }}{{ toYaml $mounts_maas_rack.volumeMounts | indent 12 }}{{ end }}
       volumes:
@@ -117,8 +126,6 @@ spec:
         - name: pod-tmp
           emptyDir: {}
 {{- if .Values.manifests.secret_ssh_key }}
-        - name: maas-ssh
-          emptyDir: {}
         - name: priv-key
           secret:
             secretName: {{ .Release.Name}}-{{ .Values.secrets.ssh_key }}
@@ -133,4 +140,14 @@ spec:
             name: maas-etc
             defaultMode: 0444
 {{ if $mounts_maas_rack.volumes }}{{ toYaml $mounts_maas_rack.volumes | indent 8 }}{{ end }}
+  volumeClaimTemplates:
+    - metadata:
+        name: rackd-state
+        annotations:
+          {{ .Values.storage.rackd.pvc.class_path }}: {{ .Values.storage.rackd.pvc.class_name }}
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: {{ .Values.storage.rackd.pvc.size }}
 {{- end }}

charts/maas/values.yaml

@@ -99,6 +99,7 @@ network:
   proxy:
     node_port:
       enabled: true
+      # Do not change the port, hardcoded in MAAS source
       port: 31800
   gui:
     node_port:
@@ -113,6 +114,13 @@ network:
     db_service: 5432
     db_service_target: 5432
 
+storage:
+  rackd:
+    pvc:
+      class_path: volume.beta.kubernetes.io/storage-class
+      class_name: general
+      size: 5Gi
+
 conf:
   ssh:
     # A SSH private key strings to mount

images/maas-rack-controller/2.3_nic_filter.patch

@@ -0,0 +1,13 @@
+diff --git a/src/provisioningserver/utils/network.py b/src/provisioningserver/utils/network.py
+index 48eb8fd..41d13a0 100644
+--- a/src/provisioningserver/utils/network.py
++++ b/src/provisioningserver/utils/network.py
+@@ -1141,7 +1141,7 @@ def get_all_interfaces_definition(annotate_with_monitored: bool=True) -> dict:
+     interfaces = {}
+     dhclient_info = get_dhclient_info()
+     iproute_info = get_ip_route()
+-    exclude_types = ["loopback", "ipip"]
++    exclude_types = ["loopback", "ipip", "ethernet"]
+     if not running_in_container():
+         exclude_types.append("ethernet")
+     ipaddr_info = {

images/maas-rack-controller/Dockerfile

@@ -11,7 +11,8 @@ RUN apt-get -qq update && \
     sudo \
     software-properties-common \
     libvirt-bin \
-    systemd
+    systemd \
+    patch
 # Don't start any optional services except for the few we need.
 
 RUN find /etc/systemd/system \
@@ -42,6 +43,10 @@ RUN systemctl enable register-rack-controller.service
 RUN mv /usr/sbin/tcpdump /usr/bin/tcpdump
 RUN ln -s /usr/bin/tcpdump /usr/sbin/tcpdump
 
+# Patch so that Calico interfaces are ignored
+COPY 2.3_nic_filter.patch /tmp/2.3_nic_filter.patch
+RUN cd /usr/lib/python3/dist-packages/provisioningserver/utils && patch network.py < /tmp/2.3_nic_filter.patch
+
 # echo journalctl logs to the container's stdout
 COPY scripts/journalctl-to-tty.service /etc/systemd/system/journalctl-to-tty.service
 RUN mkdir -p /etc/systemd/system/basic.target.wants ;\