vexxhost/openstack-operator
Code Issues Proposed changes

Remove some CRs, and deploy at the startup of the operator

Browse Source 
1. Keystone, heat, and horizon CRs are eliminated, and deploy automatically at the beginning.
2. Use a constant namespace "openstack" for the auto-deployed resources.
3. Adjust resource request.

Depends-On: https://review.opendev.org/727868/

Change-Id: I75bc8b9e73035f3ca73f00612bc4c50f42473dc3
This commit is contained in:
okozachenko 2020-05-20 02:40:38 +03:00 committed by Mohammed Naser
parent 47e04dce93
commit 10f9be432a
33 changed files with 270 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
chart/templates/clusterrole.yaml
View File

@ -23,42 +23,8 @@ rules:
- ""
resources:
- configmaps
- services
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- namespaces
- pods
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- secrets
- services
verbs:

12
chart/templates/configmap.yaml Normal file
View File

@ -0,0 +1,12 @@
apiVersion: v1
kind: ConfigMap
metadata:
namespace: {{ .Release.Namespace }}
name: operator-config
labels:
{{ include "openstack-operator.labels" . | indent 4 }}
data:
{{- with .Values.configMap }}
operator-config.yaml: |
{{ toYaml . | indent 4 }}
{{- end }}

11
chart/templates/deployment.yaml
View File

@ -21,6 +21,11 @@ spec:
- name: operator
image: vexxhost/openstack-operator:latest
command: ["/usr/local/bin/kopf"]
env:
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- with .Values.secretName }}
envFrom:
- secretRef:
@ -29,11 +34,7 @@ spec:
args:
- run
- -m
- openstack_operator.heat
- -m
- openstack_operator.horizon
- -m
- openstack_operator.keystone
- openstack_operator.operator
- -m
- openstack_operator.mcrouter
- -m

8
chart/test-values.yaml
View File

@ -1,2 +1,10 @@
---
secretName: devstack
configMap:
horizon:
ingress:
host: "horizon.vexxhost.com"
keystone:
configDir: /etc/keystone
heat:
configDir: /etc/heat

1
chart/values.yaml
View File

@ -1,3 +1,4 @@
crd:
monitoring: true
dns: true
configMap: {}

17
devstack/lib/common
View File

@ -34,16 +34,27 @@ function kubernetes_rollout_status {
kubectl rollout status deploy/$deployment
}
function kubernetes_rollout_restart {
local deployment="$1"
for i in {1..30}; do
kubectl get deploy/$deployment && break || sleep 1;
done
kubectl rollout restart deploy/$deployment
}
function proxy_pass_to_kubernetes {
local url=$1
local svc=$2
local conf=$3
local ip=$(get_kubernetes_service_ip $svc)
local apache_conf=$(apache_site_config_for $svc)
local apache_conf=$(apache_site_config_for $conf)
echo "ProxyPass \"${url}\" \"http://${ip}/\"" | sudo tee -a $apache_conf
enable_apache_site $svc
enable_apache_site $conf
restart_apache_server
}
@ -78,4 +89,4 @@ spec:
url: $3
EOF
}
export -f _get_or_create_endpoint_with_interface
export -f _get_or_create_endpoint_with_interface

20
devstack/lib/keystone
View File

@ -54,18 +54,9 @@ export -f init_keystone
# start_keystone() - Start running processes
function start_keystone {
# install keystone
cat <<EOF | kubectl apply -f-
---
apiVersion: identity.openstack.org/v1alpha1
kind: Keystone
metadata:
name: devstack
spec:
configDir: ${KEYSTONE_CONF_DIR}
EOF
# rollout keystone
kubernetes_rollout_status keystone-devstack
kubernetes_rollout_restart keystone
kubernetes_rollout_status keystone
# Get right service port for testing
local service_port=$KEYSTONE_SERVICE_PORT
@ -74,8 +65,7 @@ EOF
service_port=$KEYSTONE_SERVICE_PORT_INT
auth_protocol="http"
fi
proxy_pass_to_kubernetes /identity keystone-devstack
proxy_pass_to_kubernetes /identity keystone keystone-devstack
echo "Waiting for keystone to start..."
# Check that the keystone service is running. Even if the tls tunnel
@ -96,7 +86,7 @@ EOF
fi
# (re)start memcached to make sure we have a clean memcache.
kubectl rollout restart statefulset/memcached-devstack
kubectl rollout restart statefulset/memcached-devstack -n default
}
export -f start_keystone
@ -111,7 +101,7 @@ export -f start_keystone
# - ``KEYSTONE_SERVICE_HOST``
# - ``KEYSTONE_SERVICE_PORT``
function bootstrap_keystone {
kubectl exec deploy/keystone-devstack -- keystone-manage bootstrap \
kubectl exec deploy/keystone -- keystone-manage bootstrap \
--bootstrap-username admin \
--bootstrap-password "$ADMIN_PASSWORD" \
--bootstrap-project-name admin \

10
openstack_operator/horizon.py
View File

@ -27,9 +27,11 @@ from openstack_operator import utils
def create_secret(name, **_):
"""Create a new horizon secret"""
utils.create_or_update('horizon/secret-secretkey.yml.j2',
name=name,
secret=utils.generate_password())
res = utils.get_secret("openstack", name)
if res is None:
utils.create_or_update('horizon/secret-secretkey.yml.j2',
name=name,
secret=utils.generate_password())
@kopf.on.resume('dashboard.openstack.org', 'v1alpha1', 'horizons')
@ -78,6 +80,6 @@ def update(name, spec, **_):
utils.create_or_update('horizon/deployment.yml.j2',
config_hash=config_hash, name=name,
spec=spec, env=env)
if hasattr(spec, "ingress"):
if "ingress" in spec:
utils.create_or_update('horizon/ingress.yml.j2',
name=name, spec=spec)

14
openstack_operator/mcrouter.py
View File

@ -41,15 +41,15 @@ def create_or_resume(name, spec, **_):
"""
data = McrouterSpecEncoder().encode(spec)
utils.create_or_update('mcrouter/configmap.yml.j2',
name=name, data=data)
name=name, data=data, adopt=True)
utils.create_or_update('mcrouter/deployment.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('mcrouter/service.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('mcrouter/podmonitor.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('mcrouter/prometheusrule.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
@kopf.on.update('infrastructure.vexxhost.cloud', 'v1alpha1', 'mcrouters')
@ -61,6 +61,6 @@ def update(name, spec, **_):
"""
data = McrouterSpecEncoder().encode(spec)
utils.create_or_update('mcrouter/configmap.yml.j2',
name=name, data=data)
name=name, data=data, adopt=True)
utils.create_or_update('mcrouter/deployment.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)

12
openstack_operator/memcached.py
View File

@ -33,15 +33,15 @@ def create_or_resume(name, spec, **_):
"""
utils.create_or_update('memcached/statefulset.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('memcached/service.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('memcached/mcrouter.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('memcached/podmonitor.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('memcached/prometheusrule.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
# NOTE(mnaser): We should remove this once all deployments are no longer
# using Deployment for Memcached.
@ -58,4 +58,4 @@ def update(name, spec, **_):
"""
utils.create_or_update('memcached/statefulset.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)

2
openstack_operator/objects.py
View File

@ -26,6 +26,7 @@ from pykube.objects import ConfigMap
from pykube.objects import Deployment
from pykube.objects import HorizontalPodAutoscaler
from pykube.objects import Ingress
from pykube.objects import Namespace
from pykube.objects import NamespacedAPIObject
from pykube.objects import Pod
from pykube.objects import Secret
@ -68,6 +69,7 @@ class PrometheusRule(NamespacedAPIObject):
MAPPING = {
"v1": {
"ConfigMap": ConfigMap,
"Namespace": Namespace,
"Pod": Pod,
"Secret": Secret,
"Service": Service,

73
openstack_operator/operator.py Normal file
View File

@ -0,0 +1,73 @@
# Copyright 2020 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# pylint: disable=W0613
"""Openstack Operator
This module maintains the operator startup, it takes care of creating
the appropriate deployments, an instance of Keystone, Heat and Horizon
for the installation.
"""
import os
import kopf
from openstack_operator import heat
from openstack_operator import horizon
from openstack_operator import keystone
from openstack_operator import utils
OPERATOR_CONFIGMAP = "operator-config"
def _create_namespace():
"""Create a namespace for the operator
All resources which are managed by the operator would
be deployed on this namespace"""
utils.create_or_update('operator/namespace.yml.j2')
@kopf.on.startup()
async def startup_fn(logger, **kwargs):
"""Create several deployments at the startup of the operator
keystone, heat, and horizon
"""
namespace = os.getenv('OPERATOR_NAMESPACE')
config = utils.get_configmap(namespace, OPERATOR_CONFIGMAP)
config = utils.to_dict(config["operator-config.yaml"])
_create_namespace()
if "keystone" in config:
keystone.create_or_resume("keystone", config["keystone"])
if "horizon" in config:
horizon.create_secret("horizon")
horizon.create_or_resume("horizon", config["horizon"])
if "heat" in config:
heat.create_or_resume("heat", config["heat"])
@kopf.on.update('', 'v1', 'configmaps')
def update(name, namespace, new, **_):
"""Update the startup deployments when the operator configmap is changed
keystone, heat, and horizon
"""
if namespace == os.getenv('OPERATOR_NAMESPACE') \
and name == OPERATOR_CONFIGMAP:
config = utils.to_dict(new["data"]["operator-config.yaml"])
if "horizon" in config:
horizon.update("horizon", config["horizon"])

10
openstack_operator/rabbitmq.py
View File

@ -33,13 +33,13 @@ def create_or_resume(name, spec, **_):
"""
utils.create_or_update('rabbitmq/deployment.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('rabbitmq/service.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('rabbitmq/podmonitor.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
utils.create_or_update('rabbitmq/prometheusrule.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)
@kopf.on.update('infrastructure.vexxhost.cloud', 'v1alpha1', 'rabbitmqs')
@ -51,4 +51,4 @@ def update(name, spec, **_):
"""
utils.create_or_update('rabbitmq/deployment.yml.j2',
name=name, spec=spec)
name=name, spec=spec, adopt=True)

5
openstack_operator/templates/heat/deployment.yml.j2
View File

@ -22,7 +22,8 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: heat-{{ name }}-{{ component }}
name: heat-{{ component }}
namespace: openstack
labels:
{{ labels("heat", name, component) | indent(4) }}
spec:
@ -59,7 +60,7 @@ spec:
{% endif %}
resources:
limits:
cpu: 1000m
cpu: 500m
ephemeral-storage: 50M
memory: 512M
requests:

3
openstack_operator/templates/heat/service.yml.j2
View File

@ -16,7 +16,8 @@
apiVersion: v1
kind: Service
metadata:
name: heat-{{ name }}-{{ component }}
name: heat-{{ component }}
namespace: openstack
spec:
serviceType: ClusterIP
ports:

5
openstack_operator/templates/horizon/configmap.yml.j2
View File

@ -16,7 +16,8 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
data:
local_settings.py: |
import os
@ -30,7 +31,7 @@ data:
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': 'mcrouter-memcached-horizon-{{ name }}:11211',
'LOCATION': 'mcrouter-memcached-horizon:11211',
},
}

9
openstack_operator/templates/horizon/deployment.yml.j2
View File

@ -16,7 +16,8 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
labels:
{{ labels("horizon", name) | indent(4) }}
annotations:
@ -44,7 +45,7 @@ spec:
valueFrom:
secretKeyRef:
key: secret_key
name: horizon-{{ name }}
name: horizon
ports:
- name: horizon
containerPort: 8000
@ -60,7 +61,7 @@ spec:
ephemeral-storage: 500M
memory: 256M
requests:
cpu: 1000m
cpu: 200m
ephemeral-storage: 500M
memory: 128M
securityContext:
@ -71,7 +72,7 @@ spec:
volumes:
- configMap:
defaultMode: 420
name: horizon-{{ name }}
name: horizon
name: config
{% if 'nodeSelector' in spec %}
nodeSelector:

7
openstack_operator/templates/horizon/horizontalpodautoscaler.yml.j2
View File

@ -16,14 +16,15 @@
apiVersion: autoscaling/v1
kind: HorizontalPodAutoscaler
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
labels:
{{ labels("horizon", name) | indent(4) }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: horizon-{{ name }}
minReplicas: 3
name: horizon
minReplicas: 2
maxReplicas: 10
targetCPUUtilizationPercentage: 90

5
openstack_operator/templates/horizon/ingress.yml.j2
View File

@ -16,7 +16,8 @@
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
spec:
rules:
- host: {{ spec.ingress.host }}
@ -24,5 +25,5 @@ spec:
paths:
- path: /
backend:
serviceName: horizon-{{ name }}
serviceName: horizon
servicePort: 80

3
openstack_operator/templates/horizon/memcached.yml.j2
View File

@ -16,7 +16,8 @@
apiVersion: infrastructure.vexxhost.cloud/v1alpha1
kind: Memcached
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
labels:
{{ labels("horizon", name) | indent(4) }}
spec:

3
openstack_operator/templates/horizon/secret-secretkey.yml.j2
View File

@ -16,6 +16,7 @@
apiVersion: v1
kind: Secret
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
stringData:
secret_key: {{ secret }}

3
openstack_operator/templates/horizon/service.yml.j2
View File

@ -16,7 +16,8 @@
apiVersion: v1
kind: Service
metadata:
name: horizon-{{ name }}
name: horizon
namespace: openstack
spec:
type: ClusterIP
ports:

5
openstack_operator/templates/keystone/deployment.yml.j2
View File

@ -16,7 +16,8 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: keystone-{{ name }}
name: keystone
namespace: openstack
labels:
{{ labels("keystone", name) | indent(4) }}
spec:
@ -55,7 +56,7 @@ spec:
ephemeral-storage: 500M
memory: 1024M
requests:
cpu: 1000m
cpu: 200m
ephemeral-storage: 500M
memory: 512M
securityContext:

7
openstack_operator/templates/keystone/horizontalpodautoscaler.yml.j2
View File

@ -16,14 +16,15 @@
apiVersion: autoscaling/v1
kind: HorizontalPodAutoscaler
metadata:
name: keystone-{{ name }}
name: keystone
namespace: openstack
labels:
{{ labels("keystone", name) | indent(4) }}
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: keystone-{{ name }}
minReplicas: 3
name: keystone
minReplicas: 2
maxReplicas: 10
targetCPUUtilizationPercentage: 90

3
openstack_operator/templates/keystone/service.yml.j2
View File

@ -16,7 +16,8 @@
apiVersion: v1
kind: Service
metadata:
name: keystone-{{ name }}
name: keystone
namespace: openstack
spec:
type: ClusterIP
ports:

24
openstack_operator/templates/operator/namespace.yml.j2 Normal file
View File

@ -0,0 +1,24 @@
---
# Copyright 2020 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Namespace
metadata:
name: openstack
labels:
{{ labels("operator", "openstack") | indent(4) }}
spec:
finalizers:
- kubernetes

2
openstack_operator/templates/rabbitmq/deployment.yml.j2
View File

@ -69,7 +69,7 @@ spec:
ephemeral-storage: 1G
memory: 1G
requests:
cpu: 500m
cpu: 100m
ephemeral-storage: 500M
memory: 512M
securityContext:

4
openstack_operator/tests/unit/base.py
View File

@ -34,6 +34,7 @@ class KubernetesObjectTestCase(testtools.TestCase):
SAMPLES_PATH = 'config/samples'
SAMPLE_FILE = ''
TEMPLATE_FILE = ''
NAMESPACE_CHECK = True
@classmethod
def setUpClass(cls):
@ -48,7 +49,8 @@ class KubernetesObjectTestCase(testtools.TestCase):
def test_metadata_has_no_namespace(self):
"""Ensure that the metadata does not specify the namespace."""
self.assertNotIn("namespace", self.object["metadata"])
if self.NAMESPACE_CHECK:
self.assertNotIn("namespace", self.object["metadata"])
class KubernetesAppTestCaseMixin:

1
openstack_operator/tests/unit/test_heat.py
View File

@ -25,3 +25,4 @@ class HeatAPIDeploymentTestCase(base.DeploymentTestCase):
SAMPLE_FILE = 'orchestration_v1alpha1_heat.yaml'
TEMPLATE_FILE = 'heat/deployment.yml.j2'
NAMESPACE_CHECK = False

1
openstack_operator/tests/unit/test_horizon.py
View File

@ -25,3 +25,4 @@ class HorizonDeploymentTestCase(base.DeploymentTestCase):
SAMPLE_FILE = 'dashboard_v1alpha1_horizon.yaml'
TEMPLATE_FILE = 'horizon/deployment.yml.j2'
NAMESPACE_CHECK = False

28
openstack_operator/tests/unit/test_keystone.py Normal file
View File

@ -0,0 +1,28 @@
# Copyright 2020 VEXXHOST, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for Keystone Operator
This module contains all the tests for the Keystone operator.
"""
from openstack_operator.tests.unit import base
class KeystoneDeploymentTestCase(base.DeploymentTestCase):
"""Basic tests for the Deployment."""
SAMPLE_FILE = 'identity_v1alpha1_keystone.yaml'
TEMPLATE_FILE = 'keystone/deployment.yml.j2'
NAMESPACE_CHECK = False

32
openstack_operator/utils.py
View File

@ -141,7 +141,9 @@ def generate_yaml(template, **kwargs):
"""
doc = render_template(template, **kwargs)
kopf.adopt(doc)
if "adopt" in kwargs and kwargs["adopt"]:
kopf.adopt(doc)
return doc
@ -201,9 +203,12 @@ def get_secret(namespace, name):
api = pykube.HTTPClient(pykube.KubeConfig.from_env())
secret = objects.Secret.objects(api).filter(namespace=namespace).get(
name=name
)
try:
secret = objects.Secret.objects(api).filter(namespace=namespace).get(
name=name
)
except pykube.exceptions.ObjectDoesNotExist:
return None
return {
k: base64.b64decode(v).decode('utf-8')
@ -225,3 +230,22 @@ def get_uwsgi_env():
for key, value in UWSGI_SETTINGS.items():
res.append({'name': key, 'value': value})
return res
def get_configmap(namespace, name):
"""Retrieve a configmap from Kubernetes.
This function retrieves a configmap from Kubernetes, decodes it and passes
the value of the data
"""
api = pykube.HTTPClient(pykube.KubeConfig.from_env())
try:
config = objects.ConfigMap.objects(api).filter(
namespace=namespace
).get(name=name)
except pykube.exceptions.ObjectDoesNotExist:
return None
return config.obj["data"]

3
playbooks/functional/devstack.yaml
View File

@ -64,6 +64,9 @@
# the devstack part of the job, so we keep devstack in the main play to
# avoid zuul retrying on legitimate failures.
- hosts: all
pre_tasks:
- name: Set the context with openstack namespace
command: kubectl config set-context --current --namespace=openstack
roles:
- orchestrate-devstack