Remove some CRs, and deploy at the startup of the operator

1. Keystone, heat, and horizon CRs are eliminated, and deploy automatically at the beginning.
2. Use a constant namespace "openstack" for the auto-deployed resources.
3. Adjust resource request.

Depends-On: https://review.opendev.org/727868/

Change-Id: I75bc8b9e73035f3ca73f00612bc4c50f42473dc3
This commit is contained in:
okozachenko 2020-05-20 02:40:38 +03:00 committed by Mohammed Naser
parent 47e04dce93
commit 10f9be432a
33 changed files with 270 additions and 109 deletions

@ -23,42 +23,8 @@ rules:
- ""
resources:
- configmaps
- services
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- namespaces
- pods
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- secrets
- services
verbs:

@ -0,0 +1,12 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: {{ .Release.Namespace }}
name: operator-config
labels:
{{ include "openstack-operator.labels" . | indent 4 }}
data:
{{- with .Values.configMap }}
operator-config.yaml: |
{{ toYaml . | indent 4 }}
{{- end }}

@ -21,6 +21,11 @@ spec:
- name: operator
image: vexxhost/openstack-operator:latest
command: ["/usr/local/bin/kopf"]
env:
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- with .Values.secretName }}
envFrom:
- secretRef:
@ -29,11 +34,7 @@ spec:
args:
- run
- -m
- openstack_operator.heat
- -m
- openstack_operator.horizon
- -m
- openstack_operator.keystone
- openstack_operator.operator
- -m
- openstack_operator.mcrouter
- -m

@ -1,2 +1,10 @@
---
secretName: devstack
configMap:
horizon:
ingress:
host: "horizon.vexxhost.com"
keystone:
configDir: /etc/keystone
heat:
configDir: /etc/heat

@ -1,3 +1,4 @@
crd:
monitoring: true
dns: true
configMap: {}

@ -34,16 +34,27 @@ function kubernetes_rollout_status {
kubectl rollout status deploy/$deployment
}
function kubernetes_rollout_restart {
local deployment="$1"
for i in {1..30}; do
kubectl get deploy/$deployment && break || sleep 1;
done
kubectl rollout restart deploy/$deployment
}
function proxy_pass_to_kubernetes {
local url=$1
local svc=$2
local conf=$3
local ip=$(get_kubernetes_service_ip $svc)
local apache_conf=$(apache_site_config_for $svc)
local apache_conf=$(apache_site_config_for $conf)
echo "ProxyPass \"${url}\" \"http://${ip}/\"" | sudo tee -a $apache_conf
enable_apache_site $svc
enable_apache_site $conf
restart_apache_server
}

@ -54,18 +54,9 @@ export -f init_keystone
# start_keystone() - Start running processes
function start_keystone {
# install keystone
cat <<EOF | kubectl apply -f-
---
apiVersion: identity.openstack.org/v1alpha1
kind: Keystone
metadata:
name: devstack
spec:
configDir: ${KEYSTONE_CONF_DIR}
EOF
# rollout keystone
kubernetes_rollout_status keystone-devstack
kubernetes_rollout_restart keystone
kubernetes_rollout_status keystone
# Get right service port for testing
local service_port=$KEYSTONE_SERVICE_PORT
@ -74,8 +65,7 @@ EOF
service_port=$KEYSTONE_SERVICE_PORT_INT
auth_protocol="http"
fi
proxy_pass_to_kubernetes /identity keystone-devstack
proxy_pass_to_kubernetes /identity keystone keystone-devstack
echo "Waiting for keystone to start..."
# Check that the keystone service is running. Even if the tls tunnel
@ -96,7 +86,7 @@ EOF
fi
# (re)start memcached to make sure we have a clean memcache.
kubectl rollout restart statefulset/memcached-devstack
kubectl rollout restart statefulset/memcached-devstack -n default
}
export -f start_keystone
@ -111,7 +101,7 @@ export -f start_keystone
# - ``KEYSTONE_SERVICE_HOST``
# - ``KEYSTONE_SERVICE_PORT``
function bootstrap_keystone {
kubectl exec deploy/keystone-devstack -- keystone-manage bootstrap \
kubectl exec deploy/keystone -- keystone-manage bootstrap \
--bootstrap-username admin \
--bootstrap-password "$ADMIN_PASSWORD" \
--bootstrap-project-name admin \

@ -27,9 +27,11 @@ from openstack_operator import utils
def create_secret(name, **_):
"""Create a new horizon secret"""
utils.create_or_update('horizon/secret-secretkey.yml.j2',
name=name,
secret=utils.generate_password())
res = utils.get_secret("openstack", name)
if res is None:
utils.create_or_update('horizon/secret-secretkey.yml.j2',
name=name,
secret=utils.generate_password())
@kopf.on.resume('dashboard.openstack.org', 'v1alpha1', 'horizons')
@ -78,6 +80,6 @@ def update(name, spec, **_):
utils.create_or_update('horizon/deployment.yml.j2',
config_hash=config_hash, name=name,
spec=spec, env=env)
if hasattr(spec, "ingress"):
if "ingress" in spec:
utils.create_or_update('horizon/ingress.yml.j2',
name=name, spec=spec)

@ -41,15 +41,15 @@ def create_or_resume(name, spec, **_):
"""
data = McrouterSpecEncoder().encode(spec)
utils.create_or_update('mcrouter/configmap.yml.j2',
name=name, data=data)
name=name, data=data, adopt=True)
utils.create_or_update('mcrouter/deployment.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('mcrouter/service.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('mcrouter/podmonitor.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('mcrouter/prometheusrule.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
@kopf.on.update('infrastructure.vexxhost.cloud', 'v1alpha1', 'mcrouters')
@ -61,6 +61,6 @@ def update(name, spec, **_):
"""
data = McrouterSpecEncoder().encode(spec)
utils.create_or_update('mcrouter/configmap.yml.j2',
name=name, data=data)
name=name, data=data, adopt=True)
utils.create_or_update('mcrouter/deployment.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)

@ -33,15 +33,15 @@ def create_or_resume(name, spec, **_):
"""
utils.create_or_update('memcached/statefulset.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('memcached/service.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('memcached/mcrouter.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('memcached/podmonitor.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('memcached/prometheusrule.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
# NOTE(mnaser): We should remove this once all deployments are no longer
# using Deployment for Memcached.
@ -58,4 +58,4 @@ def update(name, spec, **_):
"""
utils.create_or_update('memcached/statefulset.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)

@ -26,6 +26,7 @@ from pykube.objects import ConfigMap
from pykube.objects import Deployment
from pykube.objects import HorizontalPodAutoscaler
from pykube.objects import Ingress
from pykube.objects import Namespace
from pykube.objects import NamespacedAPIObject
from pykube.objects import Pod
from pykube.objects import Secret
@ -68,6 +69,7 @@ class PrometheusRule(NamespacedAPIObject):
MAPPING = {
"v1": {
"ConfigMap": ConfigMap,
"Namespace": Namespace,
"Pod": Pod,
"Secret": Secret,
"Service": Service,

@ -0,0 +1,73 @@
# Copyright 2020 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=W0613
"""Openstack Operator
This module maintains the operator startup, it takes care of creating
the appropriate deployments, an instance of Keystone, Heat and Horizon
for the installation.
"""
import os
import kopf
from openstack_operator import heat
from openstack_operator import horizon
from openstack_operator import keystone
from openstack_operator import utils
OPERATOR_CONFIGMAP = "operator-config"
def _create_namespace():
"""Create a namespace for the operator
All resources which are managed by the operator would
be deployed on this namespace"""
utils.create_or_update('operator/namespace.yml.j2')
@kopf.on.startup()
async def startup_fn(logger, **kwargs):
"""Create several deployments at the startup of the operator
keystone, heat, and horizon
"""
namespace = os.getenv('OPERATOR_NAMESPACE')
config = utils.get_configmap(namespace, OPERATOR_CONFIGMAP)
config = utils.to_dict(config["operator-config.yaml"])
_create_namespace()
if "keystone" in config:
keystone.create_or_resume("keystone", config["keystone"])
if "horizon" in config:
horizon.create_secret("horizon")
horizon.create_or_resume("horizon", config["horizon"])
if "heat" in config:
heat.create_or_resume("heat", config["heat"])
@kopf.on.update('', 'v1', 'configmaps')
def update(name, namespace, new, **_):
"""Update the startup deployments when the operator configmap is changed
keystone, heat, and horizon
"""
if namespace == os.getenv('OPERATOR_NAMESPACE') \
and name == OPERATOR_CONFIGMAP:
config = utils.to_dict(new["data"]["operator-config.yaml"])
if "horizon" in config:
horizon.update("horizon", config["horizon"])

@ -33,13 +33,13 @@ def create_or_resume(name, spec, **_):
"""
utils.create_or_update('rabbitmq/deployment.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('rabbitmq/service.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('rabbitmq/podmonitor.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('rabbitmq/prometheusrule.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
@kopf.on.update('infrastructure.vexxhost.cloud', 'v1alpha1', 'rabbitmqs')
@ -51,4 +51,4 @@ def update(name, spec, **_):
"""
utils.create_or_update('rabbitmq/deployment.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)

@ -22,7 +22,8 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: heat-{{ name }}-{{ component }}
name: heat-{{ component }}
namespace: openstack
labels:
{{ labels("heat", name, component) | indent(4) }}
spec:
@ -59,7 +60,7 @@ spec:
{% endif %}
resources:
limits:
cpu: 1000m
cpu: 500m
ephemeral-storage: 50M
memory: 512M
requests:

@ -16,7 +16,8 @@
apiVersion: v1
kind: Service
metadata:
name: heat-{{ name }}-{{ component }}
name: heat-{{ component }}
namespace: openstack
spec:
serviceType: ClusterIP
ports:

@ -16,7 +16,8 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
data:
local_settings.py: |
import os
@ -30,7 +31,7 @@ data:
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': 'mcrouter-memcached-horizon-{{ name }}:11211',
'LOCATION': 'mcrouter-memcached-horizon:11211',
},
}

@ -16,7 +16,8 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
labels:
{{ labels("horizon", name) | indent(4) }}
annotations:
@ -44,7 +45,7 @@ spec:
valueFrom:
secretKeyRef:
key: secret_key
name: horizon-{{ name }}
name: horizon
ports:
- name: horizon
containerPort: 8000
@ -60,7 +61,7 @@ spec:
ephemeral-storage: 500M
memory: 256M
requests:
cpu: 1000m
cpu: 200m
ephemeral-storage: 500M
memory: 128M
securityContext:
@ -71,7 +72,7 @@ spec:
volumes:
- configMap:
defaultMode: 420
name: horizon-{{ name }}
name: horizon
name: config
{% if 'nodeSelector' in spec %}
nodeSelector:

@ -16,14 +16,15 @@
apiVersion: autoscaling/v1
kind: HorizontalPodAutoscaler
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
labels:
{{ labels("horizon", name) | indent(4) }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: horizon-{{ name }}
minReplicas: 3
name: horizon
minReplicas: 2
maxReplicas: 10
targetCPUUtilizationPercentage: 90

@ -16,7 +16,8 @@
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
spec:
rules:
- host: {{ spec.ingress.host }}
@ -24,5 +25,5 @@ spec:
paths:
- path: /
backend:
serviceName: horizon-{{ name }}
serviceName: horizon
servicePort: 80

@ -16,7 +16,8 @@
apiVersion: infrastructure.vexxhost.cloud/v1alpha1
kind: Memcached
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
labels:
{{ labels("horizon", name) | indent(4) }}
spec:

@ -16,6 +16,7 @@
apiVersion: v1
kind: Secret
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
stringData:
secret_key: {{ secret }}

@ -16,7 +16,8 @@
apiVersion: v1
kind: Service
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
spec:
type: ClusterIP
ports:

@ -16,7 +16,8 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: keystone-{{ name }}
name: keystone
namespace: openstack
labels:
{{ labels("keystone", name) | indent(4) }}
spec:
@ -55,7 +56,7 @@ spec:
ephemeral-storage: 500M
memory: 1024M
requests:
cpu: 1000m
cpu: 200m
ephemeral-storage: 500M
memory: 512M
securityContext:

@ -16,14 +16,15 @@
apiVersion: autoscaling/v1
kind: HorizontalPodAutoscaler
metadata:
name: keystone-{{ name }}
name: keystone
namespace: openstack
labels:
{{ labels("keystone", name) | indent(4) }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: keystone-{{ name }}
minReplicas: 3
name: keystone
minReplicas: 2
maxReplicas: 10
targetCPUUtilizationPercentage: 90

@ -16,7 +16,8 @@
apiVersion: v1
kind: Service
metadata:
name: keystone-{{ name }}
name: keystone
namespace: openstack
spec:
type: ClusterIP
ports:

@ -0,0 +1,24 @@
---
# Copyright 2020 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Namespace
metadata:
name: openstack
labels:
{{ labels("operator", "openstack") | indent(4) }}
spec:
finalizers:
- kubernetes

@ -69,7 +69,7 @@ spec:
ephemeral-storage: 1G
memory: 1G
requests:
cpu: 500m
cpu: 100m
ephemeral-storage: 500M
memory: 512M
securityContext:

@ -34,6 +34,7 @@ class KubernetesObjectTestCase(testtools.TestCase):
SAMPLES_PATH = 'config/samples'
SAMPLE_FILE = ''
TEMPLATE_FILE = ''
NAMESPACE_CHECK = True
@classmethod
def setUpClass(cls):
@ -48,7 +49,8 @@ class KubernetesObjectTestCase(testtools.TestCase):
def test_metadata_has_no_namespace(self):
"""Ensure that the metadata does not specify the namespace."""
self.assertNotIn("namespace", self.object["metadata"])
if self.NAMESPACE_CHECK:
self.assertNotIn("namespace", self.object["metadata"])
class KubernetesAppTestCaseMixin:

@ -25,3 +25,4 @@ class HeatAPIDeploymentTestCase(base.DeploymentTestCase):
SAMPLE_FILE = 'orchestration_v1alpha1_heat.yaml'
TEMPLATE_FILE = 'heat/deployment.yml.j2'
NAMESPACE_CHECK = False

@ -25,3 +25,4 @@ class HorizonDeploymentTestCase(base.DeploymentTestCase):
SAMPLE_FILE = 'dashboard_v1alpha1_horizon.yaml'
TEMPLATE_FILE = 'horizon/deployment.yml.j2'
NAMESPACE_CHECK = False

@ -0,0 +1,28 @@
# Copyright 2020 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for Keystone Operator
This module contains all the tests for the Keystone operator.
"""
from openstack_operator.tests.unit import base
class KeystoneDeploymentTestCase(base.DeploymentTestCase):
"""Basic tests for the Deployment."""
SAMPLE_FILE = 'identity_v1alpha1_keystone.yaml'
TEMPLATE_FILE = 'keystone/deployment.yml.j2'
NAMESPACE_CHECK = False

@ -141,7 +141,9 @@ def generate_yaml(template, **kwargs):
"""
doc = render_template(template, **kwargs)
kopf.adopt(doc)
if "adopt" in kwargs and kwargs["adopt"]:
kopf.adopt(doc)
return doc
@ -201,9 +203,12 @@ def get_secret(namespace, name):
api = pykube.HTTPClient(pykube.KubeConfig.from_env())
secret = objects.Secret.objects(api).filter(namespace=namespace).get(
name=name
)
try:
secret = objects.Secret.objects(api).filter(namespace=namespace).get(
name=name
)
except pykube.exceptions.ObjectDoesNotExist:
return None
return {
k: base64.b64decode(v).decode('utf-8')
@ -225,3 +230,22 @@ def get_uwsgi_env():
for key, value in UWSGI_SETTINGS.items():
res.append({'name': key, 'value': value})
return res
def get_configmap(namespace, name):
"""Retrieve a configmap from Kubernetes.
This function retrieves a configmap from Kubernetes, decodes it and passes
the value of the data
"""
api = pykube.HTTPClient(pykube.KubeConfig.from_env())
try:
config = objects.ConfigMap.objects(api).filter(
namespace=namespace
).get(name=name)
except pykube.exceptions.ObjectDoesNotExist:
return None
return config.obj["data"]

@ -64,6 +64,9 @@
# the devstack part of the job, so we keep devstack in the main play to
# avoid zuul retrying on legitimate failures.
- hosts: all
pre_tasks:
- name: Set the context with openstack namespace
command: kubectl config set-context --current --namespace=openstack
roles:
- orchestrate-devstack