Remove some CRs, and deploy at the startup of the operator

1. Keystone, heat, and horizon CRs are eliminated, and deploy automatically at the beginning.
2. Use a constant namespace "openstack" for the auto-deployed resources.
3. Adjust resource request.

Depends-On: https://review.opendev.org/727868/

Change-Id: I75bc8b9e73035f3ca73f00612bc4c50f42473dc3

chart/templates/clusterrole.yaml

@@ -23,42 +23,8 @@ rules:
   - ""
   resources:
   - configmaps
-  - services
+  - namespaces
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
   - pods
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
   - secrets
   - services
   verbs:

chart/templates/configmap.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: operator-config
+  labels:
+{{ include "openstack-operator.labels" . | indent 4 }}
+data:
+{{- with .Values.configMap }}
+  operator-config.yaml: |
+{{ toYaml . | indent 4 }}
+{{- end }}

chart/templates/deployment.yaml

@@ -21,6 +21,11 @@ spec:
       - name: operator
         image: vexxhost/openstack-operator:latest
         command: ["/usr/local/bin/kopf"]
+        env:
+        - name: OPERATOR_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
 {{- with .Values.secretName }}
         envFrom:
         - secretRef:
@@ -29,11 +34,7 @@ spec:
         args:
         - run
         - -m
-        - openstack_operator.heat
+        - openstack_operator.operator
-        - -m
-        - openstack_operator.horizon
-        - -m
-        - openstack_operator.keystone
         - -m
         - openstack_operator.mcrouter
         - -m

chart/test-values.yaml

@@ -1,2 +1,10 @@
 ---
 secretName: devstack
+configMap:
+  horizon:
+    ingress:
+      host: "horizon.vexxhost.com"
+  keystone:
+    configDir: /etc/keystone
+  heat:
+    configDir: /etc/heat

chart/values.yaml

@@ -1,3 +1,4 @@
 crd:
   monitoring: true
   dns: true
+configMap: {}

devstack/lib/common

@@ -34,16 +34,27 @@ function kubernetes_rollout_status {
 	kubectl rollout status deploy/$deployment
 }
 
+function kubernetes_rollout_restart {
+	local deployment="$1"
+
+	for i in {1..30}; do
+		kubectl get deploy/$deployment && break || sleep 1;
+	done
+
+	kubectl rollout restart deploy/$deployment
+}
+
 function proxy_pass_to_kubernetes {
 	local url=$1
 	local svc=$2
+	local conf=$3
 
 	local ip=$(get_kubernetes_service_ip $svc)
-	local apache_conf=$(apache_site_config_for $svc)
+	local apache_conf=$(apache_site_config_for $conf)
 
 	echo "ProxyPass \"${url}\" \"http://${ip}/\"" | sudo tee -a $apache_conf
 
-	enable_apache_site $svc
+	enable_apache_site $conf
 	restart_apache_server
 }
 

devstack/lib/keystone

@@ -54,18 +54,9 @@ export -f init_keystone
 # start_keystone() - Start running processes
 function start_keystone {
 
-	# install keystone
-	cat <<EOF | kubectl apply -f-
----
-apiVersion: identity.openstack.org/v1alpha1
-kind: Keystone
-metadata:
-  name: devstack
-spec:
-  configDir: ${KEYSTONE_CONF_DIR}
-EOF
 	# rollout keystone
-	kubernetes_rollout_status keystone-devstack
+	kubernetes_rollout_restart keystone
+	kubernetes_rollout_status keystone
 
 	# Get right service port for testing
 	local service_port=$KEYSTONE_SERVICE_PORT
@@ -74,8 +65,7 @@ EOF
 		service_port=$KEYSTONE_SERVICE_PORT_INT
 		auth_protocol="http"
 	fi
-
+	proxy_pass_to_kubernetes /identity keystone keystone-devstack
-	proxy_pass_to_kubernetes /identity keystone-devstack
 
 	echo "Waiting for keystone to start..."
 	# Check that the keystone service is running. Even if the tls tunnel
@@ -96,7 +86,7 @@ EOF
 	fi
 
 	# (re)start memcached to make sure we have a clean memcache.
-	kubectl rollout restart statefulset/memcached-devstack
+	kubectl rollout restart statefulset/memcached-devstack -n default
 }
 export -f start_keystone
 
@@ -111,7 +101,7 @@ export -f start_keystone
 # - ``KEYSTONE_SERVICE_HOST``
 # - ``KEYSTONE_SERVICE_PORT``
 function bootstrap_keystone {
-	kubectl exec deploy/keystone-devstack -- keystone-manage bootstrap \
+	kubectl exec deploy/keystone -- keystone-manage bootstrap \
 		--bootstrap-username admin \
 		--bootstrap-password "$ADMIN_PASSWORD" \
 		--bootstrap-project-name admin \

openstack_operator/horizon.py

@@ -27,6 +27,8 @@ from openstack_operator import utils
 def create_secret(name, **_):
     """Create a new horizon secret"""
 
+    res = utils.get_secret("openstack", name)
+    if res is None:
         utils.create_or_update('horizon/secret-secretkey.yml.j2',
                                name=name,
                                secret=utils.generate_password())
@@ -78,6 +80,6 @@ def update(name, spec, **_):
     utils.create_or_update('horizon/deployment.yml.j2',
                            config_hash=config_hash, name=name,
                            spec=spec, env=env)
-    if hasattr(spec, "ingress"):
+    if "ingress" in spec:
         utils.create_or_update('horizon/ingress.yml.j2',
                                name=name, spec=spec)

openstack_operator/mcrouter.py

@@ -41,15 +41,15 @@ def create_or_resume(name, spec, **_):
     """
     data = McrouterSpecEncoder().encode(spec)
     utils.create_or_update('mcrouter/configmap.yml.j2',
-                           name=name, data=data)
+                           name=name, data=data, adopt=True)
     utils.create_or_update('mcrouter/deployment.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('mcrouter/service.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('mcrouter/podmonitor.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('mcrouter/prometheusrule.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
 
 
 @kopf.on.update('infrastructure.vexxhost.cloud', 'v1alpha1', 'mcrouters')
@@ -61,6 +61,6 @@ def update(name, spec, **_):
     """
     data = McrouterSpecEncoder().encode(spec)
     utils.create_or_update('mcrouter/configmap.yml.j2',
-                           name=name, data=data)
+                           name=name, data=data, adopt=True)
     utils.create_or_update('mcrouter/deployment.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)

openstack_operator/memcached.py

@@ -33,15 +33,15 @@ def create_or_resume(name, spec, **_):
     """
 
     utils.create_or_update('memcached/statefulset.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('memcached/service.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('memcached/mcrouter.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('memcached/podmonitor.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('memcached/prometheusrule.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
 
     # NOTE(mnaser): We should remove this once all deployments are no longer
     #               using Deployment for Memcached.
@@ -58,4 +58,4 @@ def update(name, spec, **_):
     """
 
     utils.create_or_update('memcached/statefulset.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)

openstack_operator/objects.py

@@ -26,6 +26,7 @@ from pykube.objects import ConfigMap
 from pykube.objects import Deployment
 from pykube.objects import HorizontalPodAutoscaler
 from pykube.objects import Ingress
+from pykube.objects import Namespace
 from pykube.objects import NamespacedAPIObject
 from pykube.objects import Pod
 from pykube.objects import Secret
@@ -68,6 +69,7 @@ class PrometheusRule(NamespacedAPIObject):
 MAPPING = {
     "v1": {
         "ConfigMap": ConfigMap,
+        "Namespace": Namespace,
         "Pod": Pod,
         "Secret": Secret,
         "Service": Service,

openstack_operator/operator.py

@@ -0,0 +1,73 @@
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=W0613
+"""Openstack Operator
+
+This module maintains the operator startup, it takes care of creating
+the appropriate deployments, an instance of Keystone, Heat and Horizon
+ for the installation.
+"""
+
+import os
+import kopf
+
+from openstack_operator import heat
+from openstack_operator import horizon
+from openstack_operator import keystone
+from openstack_operator import utils
+
+
+OPERATOR_CONFIGMAP = "operator-config"
+
+
+def _create_namespace():
+    """Create a namespace for the operator
+
+    All resources which are managed by the operator would
+    be deployed on this namespace"""
+
+    utils.create_or_update('operator/namespace.yml.j2')
+
+
+@kopf.on.startup()
+async def startup_fn(logger, **kwargs):
+    """Create several deployments at the startup of the operator
+
+    keystone, heat, and horizon
+    """
+
+    namespace = os.getenv('OPERATOR_NAMESPACE')
+    config = utils.get_configmap(namespace, OPERATOR_CONFIGMAP)
+    config = utils.to_dict(config["operator-config.yaml"])
+    _create_namespace()
+    if "keystone" in config:
+        keystone.create_or_resume("keystone", config["keystone"])
+    if "horizon" in config:
+        horizon.create_secret("horizon")
+        horizon.create_or_resume("horizon", config["horizon"])
+    if "heat" in config:
+        heat.create_or_resume("heat", config["heat"])
+
+
+@kopf.on.update('', 'v1', 'configmaps')
+def update(name, namespace, new, **_):
+    """Update the startup deployments when the operator configmap is changed
+
+    keystone, heat, and horizon
+    """
+    if namespace == os.getenv('OPERATOR_NAMESPACE') \
+       and name == OPERATOR_CONFIGMAP:
+        config = utils.to_dict(new["data"]["operator-config.yaml"])
+        if "horizon" in config:
+            horizon.update("horizon", config["horizon"])

openstack_operator/rabbitmq.py

@@ -33,13 +33,13 @@ def create_or_resume(name, spec, **_):
     """
 
     utils.create_or_update('rabbitmq/deployment.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('rabbitmq/service.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('rabbitmq/podmonitor.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
     utils.create_or_update('rabbitmq/prometheusrule.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)
 
 
 @kopf.on.update('infrastructure.vexxhost.cloud', 'v1alpha1', 'rabbitmqs')
@@ -51,4 +51,4 @@ def update(name, spec, **_):
     """
 
     utils.create_or_update('rabbitmq/deployment.yml.j2',
-                           name=name, spec=spec)
+                           name=name, spec=spec, adopt=True)

openstack_operator/templates/heat/deployment.yml.j2

@@ -22,7 +22,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: heat-{{ name }}-{{ component }}
+  name: heat-{{ component }}
+  namespace: openstack
   labels:
     {{ labels("heat", name, component) | indent(4) }}
 spec:
@@ -59,7 +60,7 @@ spec:
         {% endif %}
         resources:
           limits:
-            cpu: 1000m
+            cpu: 500m
             ephemeral-storage: 50M
             memory: 512M
           requests:

openstack_operator/templates/heat/service.yml.j2

@@ -16,7 +16,8 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: heat-{{ name }}-{{ component }}
+  name: heat-{{ component }}
+  namespace: openstack
 spec:
   serviceType: ClusterIP
   ports:

openstack_operator/templates/horizon/configmap.yml.j2

@@ -16,7 +16,8 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: horizon-{{ name }}
+  name: horizon
+  namespace: openstack
 data:
   local_settings.py: |
     import os
@@ -30,7 +31,7 @@ data:
     CACHES = {
         'default': {
             'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
-            'LOCATION': 'mcrouter-memcached-horizon-{{ name }}:11211',
+            'LOCATION': 'mcrouter-memcached-horizon:11211',
         },
     }
 

openstack_operator/templates/horizon/deployment.yml.j2

@@ -16,7 +16,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: horizon-{{ name }}
+  name: horizon
+  namespace: openstack
   labels:
     {{ labels("horizon", name) | indent(4) }}
   annotations:
@@ -44,7 +45,7 @@ spec:
           valueFrom:
             secretKeyRef:
               key: secret_key
-              name: horizon-{{ name }}
+              name: horizon
         ports:
         - name: horizon
           containerPort: 8000
@@ -60,7 +61,7 @@ spec:
             ephemeral-storage: 500M
             memory: 256M
           requests:
-            cpu: 1000m
+            cpu: 200m
             ephemeral-storage: 500M
             memory: 128M
         securityContext:
@@ -71,7 +72,7 @@ spec:
       volumes:
       - configMap:
           defaultMode: 420
-          name: horizon-{{ name }}
+          name: horizon
         name: config
 {% if 'nodeSelector' in spec %}
       nodeSelector:

openstack_operator/templates/horizon/horizontalpodautoscaler.yml.j2

@@ -16,14 +16,15 @@
 apiVersion: autoscaling/v1
 kind: HorizontalPodAutoscaler
 metadata:
-  name: horizon-{{ name }}
+  name: horizon
+  namespace: openstack
   labels:
     {{ labels("horizon", name) | indent(4) }}
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: horizon-{{ name }}
+    name: horizon
-  minReplicas: 3
+  minReplicas: 2
   maxReplicas: 10
   targetCPUUtilizationPercentage: 90

openstack_operator/templates/horizon/ingress.yml.j2

@@ -16,7 +16,8 @@
 apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
-  name: horizon-{{ name }}
+  name: horizon
+  namespace: openstack
 spec:
   rules:
   - host: {{ spec.ingress.host }}
@@ -24,5 +25,5 @@ spec:
       paths:
       - path: /
         backend:
-          serviceName: horizon-{{ name }}
+          serviceName: horizon
           servicePort: 80

openstack_operator/templates/horizon/memcached.yml.j2

@@ -16,7 +16,8 @@
 apiVersion: infrastructure.vexxhost.cloud/v1alpha1
 kind: Memcached
 metadata:
-  name: horizon-{{ name }}
+  name: horizon
+  namespace: openstack
   labels:
     {{ labels("horizon", name) | indent(4) }}
 spec:

openstack_operator/templates/horizon/secret-secretkey.yml.j2

@@ -16,6 +16,7 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: horizon-{{ name }}
+  name: horizon
+  namespace: openstack
 stringData:
   secret_key: {{ secret }}

openstack_operator/templates/horizon/service.yml.j2

@@ -16,7 +16,8 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: horizon-{{ name }}
+  name: horizon
+  namespace: openstack
 spec:
   type: ClusterIP
   ports:

openstack_operator/templates/keystone/deployment.yml.j2

@@ -16,7 +16,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: keystone-{{ name }}
+  name: keystone
+  namespace: openstack
   labels:
     {{ labels("keystone", name) | indent(4) }}
 spec:
@@ -55,7 +56,7 @@ spec:
             ephemeral-storage: 500M
             memory: 1024M
           requests:
-            cpu: 1000m
+            cpu: 200m
             ephemeral-storage: 500M
             memory: 512M
         securityContext:

openstack_operator/templates/keystone/horizontalpodautoscaler.yml.j2

@@ -16,14 +16,15 @@
 apiVersion: autoscaling/v1
 kind: HorizontalPodAutoscaler
 metadata:
-  name: keystone-{{ name }}
+  name: keystone
+  namespace: openstack
   labels:
     {{ labels("keystone", name) | indent(4) }}
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: keystone-{{ name }}
+    name: keystone
-  minReplicas: 3
+  minReplicas: 2
   maxReplicas: 10
   targetCPUUtilizationPercentage: 90

openstack_operator/templates/keystone/service.yml.j2

@@ -16,7 +16,8 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: keystone-{{ name }}
+  name: keystone
+  namespace: openstack
 spec:
   type: ClusterIP
   ports:

openstack_operator/templates/operator/namespace.yml.j2

@@ -0,0 +1,24 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openstack
+  labels:
+    {{ labels("operator", "openstack") | indent(4) }}
+spec:
+  finalizers:
+  - kubernetes

openstack_operator/templates/rabbitmq/deployment.yml.j2

@@ -69,7 +69,7 @@ spec:
             ephemeral-storage: 1G
             memory: 1G
           requests:
-            cpu: 500m
+            cpu: 100m
             ephemeral-storage: 500M
             memory: 512M
         securityContext:

openstack_operator/tests/unit/base.py

@@ -34,6 +34,7 @@ class KubernetesObjectTestCase(testtools.TestCase):
     SAMPLES_PATH = 'config/samples'
     SAMPLE_FILE = ''
     TEMPLATE_FILE = ''
+    NAMESPACE_CHECK = True
 
     @classmethod
     def setUpClass(cls):
@@ -48,6 +49,7 @@ class KubernetesObjectTestCase(testtools.TestCase):
 
     def test_metadata_has_no_namespace(self):
         """Ensure that the metadata does not specify the namespace."""
+        if self.NAMESPACE_CHECK:
             self.assertNotIn("namespace", self.object["metadata"])
 
 

openstack_operator/tests/unit/test_heat.py

@@ -25,3 +25,4 @@ class HeatAPIDeploymentTestCase(base.DeploymentTestCase):
 
     SAMPLE_FILE = 'orchestration_v1alpha1_heat.yaml'
     TEMPLATE_FILE = 'heat/deployment.yml.j2'
+    NAMESPACE_CHECK = False

openstack_operator/tests/unit/test_horizon.py

@@ -25,3 +25,4 @@ class HorizonDeploymentTestCase(base.DeploymentTestCase):
 
     SAMPLE_FILE = 'dashboard_v1alpha1_horizon.yaml'
     TEMPLATE_FILE = 'horizon/deployment.yml.j2'
+    NAMESPACE_CHECK = False

openstack_operator/tests/unit/test_keystone.py

@@ -0,0 +1,28 @@
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for Keystone Operator
+
+This module contains all the tests for the Keystone operator.
+"""
+
+from openstack_operator.tests.unit import base
+
+
+class KeystoneDeploymentTestCase(base.DeploymentTestCase):
+    """Basic tests for the Deployment."""
+
+    SAMPLE_FILE = 'identity_v1alpha1_keystone.yaml'
+    TEMPLATE_FILE = 'keystone/deployment.yml.j2'
+    NAMESPACE_CHECK = False

openstack_operator/utils.py

@@ -141,6 +141,8 @@ def generate_yaml(template, **kwargs):
     """
 
     doc = render_template(template, **kwargs)
+
+    if "adopt" in kwargs and kwargs["adopt"]:
         kopf.adopt(doc)
 
     return doc
@@ -201,9 +203,12 @@ def get_secret(namespace, name):
 
     api = pykube.HTTPClient(pykube.KubeConfig.from_env())
 
+    try:
         secret = objects.Secret.objects(api).filter(namespace=namespace).get(
             name=name
         )
+    except pykube.exceptions.ObjectDoesNotExist:
+        return None
 
     return {
         k: base64.b64decode(v).decode('utf-8')
@@ -225,3 +230,22 @@ def get_uwsgi_env():
     for key, value in UWSGI_SETTINGS.items():
         res.append({'name': key, 'value': value})
     return res
+
+
+def get_configmap(namespace, name):
+    """Retrieve a configmap from Kubernetes.
+
+    This function retrieves a configmap from Kubernetes, decodes it and passes
+    the value of the data
+    """
+
+    api = pykube.HTTPClient(pykube.KubeConfig.from_env())
+
+    try:
+        config = objects.ConfigMap.objects(api).filter(
+            namespace=namespace
+        ).get(name=name)
+    except pykube.exceptions.ObjectDoesNotExist:
+        return None
+
+    return config.obj["data"]

playbooks/functional/devstack.yaml

@@ -64,6 +64,9 @@
 # the devstack part of the job, so we keep devstack in the main play to
 # avoid zuul retrying on legitimate failures.
 - hosts: all
+  pre_tasks:
+    - name: Set the context with openstack namespace
+      command: kubectl config set-context --current --namespace=openstack
   roles:
     - orchestrate-devstack