openstack/openstack-helm-infra
Code Issues Proposed changes

[mariadb] Add mariadb-server-primary service

Browse Source 
This PS adds mariadb-server-primary service that is getting created
and automatically updated based on the leader election process in
start.py entrypoint script.

Change-Id: I1d8a8db0ce8102e5e23f7efdeedd139726ffff28
Signed-off-by: Sergiy Markin <smarkin@mirantis.com>
This commit is contained in:
Sergiy Markin 2024-01-16 23:48:10 +00:00
parent d0b3f1c1d2
commit 07bd8c92a2
8 changed files with 227 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mariadb/Chart.yaml
View File

@ -15,7 +15,7 @@ apiVersion: v1
appVersion: v10.6.7
description: OpenStack-Helm MariaDB
name: mariadb
version: 0.2.39
version: 0.2.40
home: https://mariadb.com/kb/en/
icon: http://badges.mariadb.org/mariadb-badge-180x60.png
sources:

143
mariadb/templates/bin/_start.py.tpl
View File

@ -80,6 +80,10 @@ if check_env_var("STATE_CONFIGMAP"):
state_configmap_name = os.environ['STATE_CONFIGMAP']
logger.info("Will use \"{0}\" configmap for cluster state info".format(
state_configmap_name))
if check_env_var("PRIMARY_SERVICE_NAME"):
primary_service_name = os.environ['PRIMARY_SERVICE_NAME']
logger.info("Will use \"{0}\" service as primary".format(
primary_service_name))
if check_env_var("POD_NAMESPACE"):
pod_namespace = os.environ['POD_NAMESPACE']
if check_env_var("DIRECT_SVC_NAME"):
@ -92,6 +96,8 @@ if check_env_var("DISCOVERY_DOMAIN"):
discovery_domain = os.environ['DISCOVERY_DOMAIN']
if check_env_var("WSREP_PORT"):
wsrep_port = os.environ['WSREP_PORT']
if check_env_var("MARIADB_PORT"):
mariadb_port = int(os.environ['MARIADB_PORT'])
if check_env_var("MYSQL_DBADMIN_USERNAME"):
mysql_dbadmin_username = os.environ['MYSQL_DBADMIN_USERNAME']
if check_env_var("MYSQL_DBADMIN_PASSWORD"):
@ -115,7 +121,8 @@ if mysql_dbadmin_username == mysql_dbsst_username:
sys.exit(1)
# Set some variables for tuneables
cluster_leader_ttl = 120
if check_env_var("CLUSTER_LEADER_TTL"):
cluster_leader_ttl = int(os.environ['CLUSTER_LEADER_TTL'])
state_configmap_update_period = 10
default_sleep = 20
@ -138,6 +145,25 @@ def ensure_state_configmap(pod_namespace, configmap_name, configmap_body):
return False
def ensure_primary_service(pod_namespace, service_name, service_body):
"""Ensure the primary service exists.
Keyword arguments:
pod_namespace -- the namespace to house the service
service_name -- the service name
service_body -- the service body
"""
try:
k8s_api_instance.read_namespaced_service(
name=service_name, namespace=pod_namespace)
return True
except:
k8s_api_instance.create_namespaced_service(
namespace=pod_namespace, body=service_body)
return False
def run_cmd_with_logging(popenargs,
logger,
@ -388,6 +414,60 @@ def set_configmap_data(key, value):
return safe_update_configmap(
configmap_dict=configmap_dict, configmap_patch=configmap_patch)
def safe_update_service(service_dict, service_patch):
"""Update a service with locking.
Keyword arguments:
service_dict -- a dict representing the service to be patched
service_patch -- a dict containign the patch
"""
logger.debug("Safe Patching service")
# NOTE(portdirect): Explictly set the resource version we are patching to
# ensure nothing else has modified the service since we read it.
service_patch['metadata']['resourceVersion'] = service_dict[
'metadata']['resource_version']
# Retry up to 8 times in case of 409 only. Each retry has a ~1 second
# sleep in between so do not want to exceed the roughly 10 second
# write interval per cm update.
for i in range(8):
try:
api_response = k8s_api_instance.patch_namespaced_service(
name=primary_service_name,
namespace=pod_namespace,
body=service_patch)
return True
except kubernetes.client.rest.ApiException as error:
if error.status == 409:
# This status code indicates a collision trying to write to the
# service while another instance is also trying the same.
logger.warning("Collision writing service: {0}".format(error))
# This often happens when the replicas were started at the same
# time, and tends to be persistent. Sleep with some random
# jitter value briefly to break the synchronization.
naptime = secretsGen.uniform(0.8,1.2)
time.sleep(naptime)
else:
logger.error("Failed to set service: {0}".format(error))
return error
logger.info("Retry writing service attempt={0} sleep={1}".format(
i+1, naptime))
return True
def set_primary_service_spec(key, value):
"""Update a service's endpoint via patching.
Keyword arguments:
key -- the key to be patched
value -- the value to give the key
"""
logger.debug("Setting service spec.selector key={0} to value={1}".format(key, value))
service_dict = k8s_api_instance.read_namespaced_service(
name=primary_service_name, namespace=pod_namespace).to_dict()
service_patch = {'spec': {'selector': {}}, 'metadata': {}}
service_patch['spec']['selector'][key] = value
return safe_update_service(
service_dict=service_dict, service_patch=service_patch)
def get_configmap_value(key, type='data'):
"""Get a configmap's key's value.
@ -469,6 +549,35 @@ def get_cluster_state():
pod_namespace=pod_namespace,
configmap_name=state_configmap_name,
configmap_body=initial_configmap_body)
initial_primary_service_body = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": primary_service_name,
},
"spec": {
"ports": [
{
"name": "mysql",
"port": mariadb_port
}
],
"selector": {
"application": "mariadb",
"component": "server",
"statefulset.kubernetes.io/pod-name": leader
}
}
}
if ensure_primary_service(
pod_namespace=pod_namespace,
service_name=primary_service_name,
service_body=initial_primary_service_body):
logger.info("Service {0} already exists".format(primary_service_name))
else:
logger.info("Service {0} has been successfully created".format(primary_service_name))
return state
@ -480,6 +589,38 @@ def declare_myself_cluster_leader():
leader_expiry = "{0}Z".format(leader_expiry_raw.isoformat("T"))
set_configmap_annotation(
key='openstackhelm.openstack.org/leader.node', value=local_hostname)
logger.info("Setting primary_service's spec.selector to {0}".format(local_hostname))
try:
set_primary_service_spec(
key='statefulset.kubernetes.io/pod-name', value=local_hostname)
except:
initial_primary_service_body = {
"apiVersion": "v1",
"kind": "Service",
"metadata": {
"name": primary_service_name,
},
"spec": {
"ports": [
{
"name": "mysql",
"port": mariadb_port
}
],
"selector": {
"application": "mariadb",
"component": "server",
"statefulset.kubernetes.io/pod-name": local_hostname
}
}
}
if ensure_primary_service(
pod_namespace=pod_namespace,
service_name=primary_service_name,
service_body=initial_primary_service_body):
logger.info("Service {0} already exists".format(primary_service_name))
else:
logger.info("Service {0} has been successfully created".format(primary_service_name))
set_configmap_annotation(
key='openstackhelm.openstack.org/leader.expiry', value=leader_expiry)

29
mariadb/templates/statefulset.yaml
View File

@ -47,6 +47,29 @@ rules:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- services
verbs:
- create
- apiGroups:
- ""
resourceNames:
- {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
resources:
- services
verbs:
- get
- patch
- apiGroups:
- ""
resourceNames:
- {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
resources:
- endpoints
verbs:
- get
- apiGroups:
- ""
resourceNames:
@ -165,6 +188,12 @@ spec:
value: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: STATE_CONFIGMAP
value: {{ printf "%s-%s" .deployment_name "mariadb-state" | quote }}
- name: PRIMARY_SERVICE_NAME
value: {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
- name: CLUSTER_LEADER_TTL
value: {{ .Values.conf.galera.cluster_leader_ttl | quote }}
- name: MARIADB_PORT
value: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
- name: MYSQL_DBADMIN_USERNAME
value: {{ .Values.endpoints.oslo_db.auth.admin.username }}
- name: MYSQL_DBADMIN_PASSWORD

3
mariadb/values.yaml
View File

@ -362,6 +362,8 @@ conf:
lock_expire_after: 7200
retry_after: 3600
container_name: throttle-backups-manager
galera:
cluster_leader_ttl: 120
database:
mysql_histfile: "/dev/null"
my: |
@ -603,6 +605,7 @@ endpoints:
direct: mariadb-server
discovery: mariadb-discovery
error_pages: mariadb-ingress-error-pages
primary: mariadb-server-primary
host_fqdn_override:
default: null
path: null

21
mariadb/values_overrides/primary-service.yaml Normal file
View File

@ -0,0 +1,21 @@
---
manifests:
deployment_ingress: false
deployment_error: false
service_ingress: false
configmap_ingress_conf: false
configmap_ingress_etc: false
service_error: false
volume:
size: 1Gi
backup:
size: 1Gi
conf:
galera:
cluster_leader_ttl: 10
endpoints:
oslo_db:
hosts:
default: mariadb
primary: mariadb
...

1
releasenotes/notes/mariadb.yaml
View File

@ -55,4 +55,5 @@ mariadb:
- 0.2.37 Backups verification improvements
- 0.2.38 Added throttling remote backups
- 0.2.39 Template changes for image 1.9 compatibility
- 0.2.40 Start.py allows to create mariadb-service-primary service and endpoint
...

28
zuul.d/jobs.yaml
View File

@ -266,6 +266,34 @@
- ./tools/deployment/openstack-support/120-powerdns.sh
- ./tools/deployment/openstack-support/130-cinder.sh
- job:
name: openstack-helm-infra-openstack-support-mariadb-service-primary
parent: openstack-helm-infra-deploy
nodeset: openstack-helm-1node-ubuntu_focal
vars:
osh_params:
openstack_release: "2023.1"
container_distro_name: ubuntu
container_distro_version: focal
feature_gates: "ssl,primary-service"
gate_scripts:
- ./tools/deployment/openstack-support/000-prepare-k8s.sh
- ./tools/deployment/openstack-support/007-namespace-config.sh
- ./tools/deployment/openstack-support/010-ingress.sh
- ./tools/deployment/ceph/ceph.sh
- ./tools/deployment/openstack-support/025-ceph-ns-activate.sh
- ./tools/deployment/openstack-support/030-rabbitmq.sh
- ./tools/deployment/openstack-support/070-mariadb.sh
- ./tools/deployment/openstack-support/040-memcached.sh
- ./tools/deployment/openstack-support/051-libvirt-ssl.sh
- ./tools/deployment/openstack-support/060-openvswitch.sh
- ./tools/deployment/common/setup-client.sh
- ./tools/deployment/openstack-support/090-keystone.sh
- ./tools/deployment/openstack-support/100-ceph-radosgateway.sh
- ./tools/deployment/openstack-support/110-openstack-exporter.sh
- ./tools/deployment/openstack-support/120-powerdns.sh
- ./tools/deployment/openstack-support/130-cinder.sh
- job:
name: openstack-helm-infra-mariadb-operator

2
zuul.d/project.yaml
View File

@ -29,6 +29,7 @@
- openstack-helm-infra-openstack-support-ssl
- openstack-helm-infra-metacontroller
- openstack-helm-infra-mariadb-operator
- openstack-helm-infra-openstack-support-mariadb-service-primary
gate:
jobs:
- openstack-helm-lint
@ -38,6 +39,7 @@
- openstack-helm-infra-openstack-support
- openstack-helm-infra-openstack-support-rook
- openstack-helm-infra-openstack-support-ssl
- openstack-helm-infra-openstack-support-mariadb-service-primary
post:
jobs:
- publish-openstack-helm-charts