Add mysql cluster to keystone deployment

- Use mysqlCluster, mysql.presslabs.org/v1alpha1 CR
- Remove keycloak hack in the magnum devstack

Change-Id: I28bcf29da2b1bfb37d23ca5ff5a1851ff145819f

chart/templates/clusterrole.yaml

@@ -288,3 +288,23 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - mysql.presslabs.org
+  resources:
+  - mysqlclusters
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - mysql.presslabs.org
+  resources:
+  - mysqlclusters/status
+  verbs:
+  - get
+  - patch
+  - update

chart/test-values.yaml

@@ -19,7 +19,8 @@ configMap:
     ingress:
       host: "horizon.vexxhost.com"
   keystone:
-    configDir: /etc/keystone
+    mysql:
+      size: 10Gi
   heat:
     configDir: /etc/heat
   magnum:

config/samples/operator-config.yaml

@@ -9,8 +9,7 @@ data:
     horizon:
       ingress:
         host: "horizon.vexxhost.com"
-    keystone:
-      configDir: /etc/keystone
+    keystone: {}
     heat:
       configDir: /etc/heat
       ingress:

devstack/lib/keystone

@@ -67,7 +67,11 @@ function configure_keystone {
         iniset $KEYSTONE_CONF token provider $KEYSTONE_TOKEN_FORMAT
     fi
 
-    iniset $KEYSTONE_CONF database connection `database_connection_url keystone`
+    # Get mysql password
+    KEYSTONE_DATABASE_USER=$(get_data_from_secret keystone-mysql openstack USER)
+    KEYSTONE_DATABASE_PASSWORD=$(get_data_from_secret keystone-mysql openstack PASSWORD)
+    KEYSTONE_DATABASE_NAME=$(get_data_from_secret keystone-mysql openstack DATABASE)
+    iniset $KEYSTONE_CONF database connection "mysql+pymysql://$KEYSTONE_DATABASE_USER:$KEYSTONE_DATABASE_PASSWORD@keystone-mysql/$KEYSTONE_DATABASE_NAME?charset=utf8"
 
     # Set up logging
     if [ "$SYSLOG" != "False" ]; then
@@ -108,8 +112,7 @@ function configure_keystone {
 
 # init_keystone() - Initialize databases, etc.
 function init_keystone {
-	kubectl create secret generic keystone-config --from-file=/etc/keystone/keystone.conf -n openstack
-
+    kubectl create secret generic keystone-config --from-file=/etc/keystone/keystone.conf -n openstack
 	# NOTE(mnaser): Permissions here are bad but it's temporary so we don't care as much.
 	sudo chmod -Rv 777 /etc/keystone
 

devstack/plugin-magnum.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+#
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Save trace setting
+XTRACE=$(set +o | grep xtrace)
+set -o xtrace
+
+echo_summary "magnum's plugin.sh was called..."
+source $DEST/magnum/devstack/lib/magnum
+(set -o posix; set)
+
+if is_service_enabled magnum-api magnum-cond; then
+    if [[ "$1" == "stack" && "$2" == "install" ]]; then
+        echo_summary "Installing magnum"
+        install_magnum
+
+        MAGNUM_GUEST_IMAGE_URL=${MAGNUM_GUEST_IMAGE_URL:-"https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/31.20200323.3.2/x86_64/fedora-coreos-31.20200323.3.2-openstack.x86_64.qcow2.xz"}
+        IMAGE_URLS+=",${MAGNUM_GUEST_IMAGE_URL}"
+
+        LIBS_FROM_GIT="${LIBS_FROM_GIT},python-magnumclient"
+
+        install_magnumclient
+        cleanup_magnum
+    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
+        echo_summary "Configuring magnum"
+        configure_magnum
+
+        if is_service_enabled key; then
+            create_magnum_accounts
+        fi
+
+    elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
+        # Initialize magnum
+        init_magnum
+        magnum_register_image
+        magnum_configure_flavor
+
+        # Start the magnum API and magnum taskmgr components
+        echo_summary "Starting magnum"
+        start_magnum
+
+        configure_iptables_magnum
+        configure_apache_magnum
+    fi
+
+    if [[ "$1" == "unstack" ]]; then
+        stop_magnum
+    fi
+
+    if [[ "$1" == "clean" ]]; then
+        cleanup_magnum
+    fi
+fi
+
+# Restore xtrace
+$XTRACE

openstack_operator/database.py

@@ -0,0 +1,40 @@
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""database Operator
+
+This module contains a few common functions for database management
+"""
+
+from openstack_operator import utils
+
+
+def ensure_mysql_cluster(name, spec):
+    """Create or update mysql cluster"""
+
+    config = utils.get_secret("openstack", name + "-mysql")
+    if config is None:
+        root_password = utils.generate_password()
+        password = utils.generate_password()
+        user = name
+        database = name
+        utils.create_or_update('mysqlcluster/secret-mysqlcluster.yml.j2',
+                               name=name, user=user,
+                               database=database, password=password,
+                               rootPassword=root_password)
+        config = utils.get_secret("openstack", name + "-mysql")
+
+    utils.create_or_update('mysqlcluster/mysqlcluster.yml.j2',
+                           server_side=False, name=name, spec=spec)
+    return config

openstack_operator/keystone.py

@@ -23,6 +23,7 @@ import kopf
 
 from cryptography import fernet
 
+from openstack_operator import database
 from openstack_operator import filters
 from openstack_operator import utils
 
@@ -107,10 +108,16 @@ def create_or_resume(name, spec, **_):
                            region_name=region_name,
                            username=username)
     # (TODO)Replace the current admin url
+
+    if "mysql" not in spec:
+        spec["mysql"] = {}
+    database.ensure_mysql_cluster("keystone", spec["mysql"])
+
+    utils.create_or_update('keystone/memcached.yml.j2', spec=spec)
+
     utils.create_or_update('keystone/daemonset.yml.j2',
                            name=name, spec=spec,
                            config_hash=config_hash)
-    utils.create_or_update('keystone/memcached.yml.j2', spec=spec)
     utils.create_or_update('keystone/service.yml.j2',
                            name=name, spec=spec)
     if "ingress" in spec:

openstack_operator/objects.py

@@ -52,6 +52,14 @@ class Memcached(NamespacedAPIObject):
     kind = "Memcached"
 
 
+class MysqlCluster(NamespacedAPIObject):
+    """Mysql Cluster Kubernetes object"""
+
+    version = "mysql.presslabs.org/v1alpha1"
+    endpoint = "mysqlclusters"
+    kind = "MysqlCluster"
+
+
 class PodMonitor(NamespacedAPIObject):
     """PodMonitor Kubernetes object"""
 
@@ -109,5 +117,8 @@ MAPPING = {
     },
     "networking.k8s.io/v1beta1": {
         "Ingress": Ingress
+    },
+    "mysql.presslabs.org/v1alpha1": {
+        "MysqlCluster": MysqlCluster
     }
 }

openstack_operator/templates/keystone/daemonset.yml.j2

@@ -122,8 +122,8 @@ spec:
           runAsUser: 65534
           runAsGroup: 65534
         volumeMounts:
-        - mountPath: /etc/keystone/
-          name: config
+        - name: config
+          mountPath: /etc/keystone
         - name: fernet-keys
           mountPath: /etc/keystone/fernet-keys
         - name: credential-keys

openstack_operator/templates/mysqlcluster/mysqlcluster.yml.j2

@@ -0,0 +1,51 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: mysql.presslabs.org/v1alpha1
+kind: MysqlCluster
+metadata:
+  name: {{ name }}
+  namespace: openstack
+  labels:
+    {{ labels(name) | indent(4) }}
+spec:
+  replicas: 2
+  secretName: {{ name }}-mysql
+  {% if "mysqlConf" in spec %}
+  mysqlConf:
+    {{ spec.mysqlConf | to_yaml | indent(4) }}
+  {% endif %}
+  podSpec:
+    affinity:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+        - labelSelector:
+            matchLabels:
+              mysql.presslabs.org/cluster: {{ name }}
+          topologyKey: kubernetes.io/hostname
+    nodeSelector:
+      node-role.kubernetes.io/master: ""
+    tolerations:
+    - effect: NoSchedule
+      key: node-role.kubernetes.io/master
+  {% if "volumeSpec" in spec %}
+  volumeSpec:
+    persistentVolumeClaim:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: {{ spec.size | default("10Gi", true) }}
+  {% endif %}

openstack_operator/templates/mysqlcluster/secret-mysqlcluster.yml.j2

@@ -0,0 +1,25 @@
+---
+# Copyright 2020 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ name }}-mysql
+  namespace: openstack
+stringData:
+  ROOT_PASSWORD: {{ rootPassword }}
+  USER: {{ user }}
+  PASSWORD: {{ password }}
+  DATABASE: {{ database }}

openstack_operator/utils.py

@@ -236,7 +236,9 @@ def ensure_secret(namespace, name):
 
 
 def generate_hash(dictionary):
-    """Generate a hash from a dictionary, return None if dictionary is empty"""
+    """Generate a hash from a dictionary, return None
+    if dictionary is empty"""
+
     if not dictionary:
         return None
     return hash(json.dumps(dictionary))

playbooks/functional/devstack.yaml

@@ -77,6 +77,9 @@
     - name: Override magnum lib functions
       become: true
       command: rsync -av src/opendev.org/vexxhost/openstack-operator/devstack/lib/magnum /opt/stack/magnum/devstack/lib/magnum
+    - name: Override magnum plugin.sh
+      become: true
+      command: rsync -av src/opendev.org/vexxhost/openstack-operator/devstack/plugin-magnum.sh /opt/stack/magnum/devstack/plugin.sh
 
 # Changes that run through devstack-tempest are likely to have an impact on
 # the devstack part of the job, so we keep devstack in the main play to

playbooks/functional/run.yaml

@@ -34,7 +34,17 @@
           OS_PASSWORD: secretadmin
           OS_REGION_NAME: RegionOne
         EOF
+    - name: Copy mysql operator into devstack working directory
+      git:
+        repo: https://github.com/presslabs/mysql-operator
+        dest: /opt/stack/mysql-operator
+      become: true
   roles:
+    - role: helm-template
+      vars:
+        helm_release_name: mysql-operator
+        helm_chart: /opt/stack/mysql-operator/charts/mysql-operator
+        helm_values_file: /opt/stack/mysql-operator/charts/mysql-operator/values.yaml
     - role: helm-template
       vars:
         helm_release_name: openstack-operator