Switch Memcached to use StatefulSet

While Memcached is not stateful, it is very useful to have a stable DNS
which will allow us to simplify the Mcrouter resource configuration.

Change-Id: I0776875ed24c8c8681fe4efd334a7e911c0ae4ce

chart/templates/clusterrole.yaml

@@ -10,6 +10,7 @@ rules:
   - apps
   resources:
   - deployments
+  - statefulsets
   verbs:
   - create
   - delete

openstack_operator/memcached.py

@@ -32,13 +32,22 @@ def create_or_resume(name, spec, **_):
     start the service up for the first time.
     """
 
-    utils.create_or_update('memcached/deployment.yml.j2',
+    utils.create_or_update('memcached/statefulset.yml.j2',
+                           name=name, spec=spec)
+    utils.create_or_update('memcached/service.yml.j2',
+                           name=name, spec=spec)
+    utils.create_or_update('memcached/mcrouter.yml.j2',
                            name=name, spec=spec)
     utils.create_or_update('memcached/podmonitor.yml.j2',
                            name=name, spec=spec)
     utils.create_or_update('memcached/prometheusrule.yml.j2',
                            name=name, spec=spec)
 
+    # NOTE(mnaser): We should remove this once all deployments are no longer
+    #               using Deployment for Memcached.
+    utils.ensure_absent('memcached/deployment.yml.j2',
+                        name=name, spec=spec)
+
 
 @kopf.on.update('infrastructure.vexxhost.cloud', 'v1alpha1', 'memcacheds')
 def update(name, spec, **_):
@@ -48,26 +57,5 @@ def update(name, spec, **_):
     changes that happen within it.
     """
 
-    utils.create_or_update('memcached/deployment.yml.j2',
+    utils.create_or_update('memcached/statefulset.yml.j2',
                            name=name, spec=spec)
-
-
-@kopf.on.event('apps', 'v1', 'deployments', labels={
-    'app.kubernetes.io/managed-by': 'openstack-operator',
-    'app.kubernetes.io/name': 'memcached',
-})
-def deployment_event(namespace, meta, spec, **_):
-    """Create and re-sync Mcrouter instances
-
-    This function takes care of watching for the readyReplicas on the
-    Deployments for Memcached to both update and synchronize the Mcrouter.
-    """
-
-    name = meta.get('labels', {}).get('app.kubernetes.io/instance')
-    selector = spec.get('selector', {}).get('matchLabels', {})
-    servers = utils.get_ready_pod_ips(namespace, selector)
-    memcacheds = ["%s:11211" % s for s in servers]
-
-    utils.create_or_update('memcached/mcrouter.yml.j2',
-                           name=name, servers=memcacheds,
-                           spec=spec.get('template', {}).get('spec', {}))

openstack_operator/objects.py

@@ -27,6 +27,7 @@ from pykube.objects import Deployment
 from pykube.objects import NamespacedAPIObject
 from pykube.objects import Pod
 from pykube.objects import Service
+from pykube.objects import StatefulSet
 
 
 class Mcrouter(NamespacedAPIObject):
@@ -61,6 +62,7 @@ MAPPING = {
     },
     "apps/v1": {
         "Deployment": Deployment,
+        "StatefulSet": StatefulSet,
     },
     "infrastructure.vexxhost.cloud/v1alpha1": {
         "Mcrouter": Mcrouter,

openstack_operator/templates/memcached/mcrouter.yml.j2

@@ -21,7 +21,8 @@ spec:
   pools:
     default:
       servers:
-        {{ servers | to_yaml | indent(8) }}
+        - memcached-{{ name }}-0.memcached-{{ name }}:11211
+        - memcached-{{ name }}-1.memcached-{{ name }}:11211
   route: PoolRoute|default
 {% if 'nodeSelector' in spec %}
   nodeSelector:

openstack_operator/templates/memcached/service.yml.j2

@@ -0,0 +1,27 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: memcached-{{ name }}
+spec:
+  clusterIP: None
+  ports:
+  - name: memcached
+    port: 11211
+    targetPort: memcached
+  selector:
+    {{ labels("memcached", name) | indent(4) }}

openstack_operator/templates/memcached/statefulset.yml.j2

@@ -0,0 +1,90 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: memcached-{{ name }}
+  labels:
+    {{ labels("memcached", name) | indent(4) }}
+spec:
+  replicas: 2
+  serviceName: memcached-{{ name }}
+  selector:
+    matchLabels:
+      {{ labels("memcached", name) | indent(6) }}
+  template:
+    metadata:
+      labels:
+        {{ labels("memcached", name) | indent(8) }}
+    spec:
+      containers:
+      - name: memcached
+        image: vexxhost/memcached:latest
+        args: ["-m", "{{ (spec.megabytes / 2) | int }}"]
+        imagePullPolicy: Always
+        ports:
+        - name: memcached
+          containerPort: 11211
+        livenessProbe:
+          tcpSocket:
+            port: memcached
+        readinessProbe:
+          tcpSocket:
+            port: memcached
+        resources:
+          limits:
+            cpu: 50m
+            ephemeral-storage: 50M
+            memory: {{ (spec.megabytes / 2) | int + 64 }}M
+          requests:
+            cpu: 10m
+            ephemeral-storage: 50M
+            memory: {{ (spec.megabytes / 2) | int }}M
+        securityContext:
+          runAsUser: 1001
+      - name: exporter
+        image: vexxhost/memcached_exporter:latest
+        imagePullPolicy: Always
+        ports:
+        - name: metrics
+          containerPort: 9150
+        livenessProbe:
+          httpGet:
+            path: /metrics
+            port: metrics
+        readinessProbe:
+          httpGet:
+            path: /metrics
+            port: metrics
+        resources:
+          limits:
+            cpu: 100m
+            ephemeral-storage: 10M
+            memory: 64Mi
+          requests:
+            cpu: 50m
+            ephemeral-storage: 10M
+            memory: 32Mi
+        securityContext:
+          runAsUser: 1001
+{% if 'nodeSelector' in spec %}
+      nodeSelector:
+        {{ spec.nodeSelector | to_yaml | indent(8) }}
+{% endif %}
+{% if 'tolerations' in spec %}
+      tolerations:
+        {{ spec.tolerations | to_yaml | indent(8) }}
+{% endif %}

openstack_operator/tests/unit/test_memcached.py

@@ -27,40 +27,13 @@ from oslotest import base
 from openstack_operator import memcached
 
 
-class MemcachedListTestCase(base.BaseTestCase):
+class MemcachedOperatorTestCase(base.BaseTestCase):
-    """Tests for determining server list."""
+    """Basic tests for the operator."""
 
-    @mock.patch.object(memcached.utils, 'get_ready_pod_ips')
     @mock.patch.object(memcached.utils, 'create_or_update')
-    def test_with_no_ips(self, mock_create, mock_get_ready_pods):
+    @mock.patch.object(memcached.utils, 'ensure_absent')
-        """Test a deployment with no ready pods."""
+    def test_ensure_deployment_removal(self, mock_ensure_absent, _):
-
+        """Test that we remove the old deployment"""
-        mock_get_ready_pods.return_value = []
+        memcached.create_or_resume("foo", {})
-        memcached.deployment_event("default", {}, {})
+        mock_ensure_absent.assert_called_once_with(
-
+            'memcached/deployment.yml.j2', name="foo", spec={})
-        mock_create.assert_called_once_with('memcached/mcrouter.yml.j2',
-                                            name=None, servers=[], spec={})
-
-    @mock.patch.object(memcached.utils, 'get_ready_pod_ips')
-    @mock.patch.object(memcached.utils, 'create_or_update')
-    def test_with_single_ip(self, mock_create, mock_get_ready_pods):
-        """Test a deployment with a single ready pod."""
-
-        mock_get_ready_pods.return_value = ['1.1.1.1']
-        memcached.deployment_event("default", {}, {})
-
-        mock_create.assert_called_once_with(
-            'memcached/mcrouter.yml.j2', name=None,
-            servers=['1.1.1.1:11211'], spec={})
-
-    @mock.patch.object(memcached.utils, 'get_ready_pod_ips')
-    @mock.patch.object(memcached.utils, 'create_or_update')
-    def test_multiple_ips(self, mock_create, mock_get_ready_pods):
-        """Test a deployment with a multiple ready pods."""
-
-        mock_get_ready_pods.return_value = ['1.1.1.1', '2.2.2.2']
-        memcached.deployment_event("default", {}, {})
-
-        mock_create.assert_called_once_with(
-            'memcached/mcrouter.yml.j2', name=None,
-            servers=['1.1.1.1:11211', '2.2.2.2:11211'], spec={})

openstack_operator/utils.py

@@ -77,6 +77,17 @@ def create_or_update(template, **kwargs):
         resource.create()
 
 
+def ensure_absent(template, **kwargs):
+    """Ensure a Kubernetes resource bound to a template is deleted
+
+    This function gets a template and makes sure that the object doesn't
+    exist on the remote cluster.
+    """
+
+    resource = generate_object(template, **kwargs)
+    resource.delete()
+
+
 def generate_yaml(template, **kwargs):
     """Generate dictionary from YAML template.