From 07bd8c92a259557d07119525c85bea4b8fc6006e Mon Sep 17 00:00:00 2001 From: Sergiy Markin Date: Tue, 16 Jan 2024 23:48:10 +0000 Subject: [PATCH] [mariadb] Add mariadb-server-primary service This PS adds mariadb-server-primary service that is getting created and automatically updated based on the leader election process in start.py entrypoint script. Change-Id: I1d8a8db0ce8102e5e23f7efdeedd139726ffff28 Signed-off-by: Sergiy Markin --- mariadb/Chart.yaml | 2 +- mariadb/templates/bin/_start.py.tpl | 143 +++++++++++++++++- mariadb/templates/statefulset.yaml | 29 ++++ mariadb/values.yaml | 3 + mariadb/values_overrides/primary-service.yaml | 21 +++ releasenotes/notes/mariadb.yaml | 1 + zuul.d/jobs.yaml | 28 ++++ zuul.d/project.yaml | 2 + 8 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 mariadb/values_overrides/primary-service.yaml diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml index 56ca9c2aa..ebe49e9aa 100644 --- a/mariadb/Chart.yaml +++ b/mariadb/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v10.6.7 description: OpenStack-Helm MariaDB name: mariadb -version: 0.2.39 +version: 0.2.40 home: https://mariadb.com/kb/en/ icon: http://badges.mariadb.org/mariadb-badge-180x60.png sources: diff --git a/mariadb/templates/bin/_start.py.tpl b/mariadb/templates/bin/_start.py.tpl index db36168a5..aae1294ca 100644 --- a/mariadb/templates/bin/_start.py.tpl +++ b/mariadb/templates/bin/_start.py.tpl @@ -80,6 +80,10 @@ if check_env_var("STATE_CONFIGMAP"): state_configmap_name = os.environ['STATE_CONFIGMAP'] logger.info("Will use \"{0}\" configmap for cluster state info".format( state_configmap_name)) +if check_env_var("PRIMARY_SERVICE_NAME"): + primary_service_name = os.environ['PRIMARY_SERVICE_NAME'] + logger.info("Will use \"{0}\" service as primary".format( + primary_service_name)) if check_env_var("POD_NAMESPACE"): pod_namespace = os.environ['POD_NAMESPACE'] if check_env_var("DIRECT_SVC_NAME"): @@ -92,6 +96,8 @@ if check_env_var("DISCOVERY_DOMAIN"): discovery_domain = os.environ['DISCOVERY_DOMAIN'] if check_env_var("WSREP_PORT"): wsrep_port = os.environ['WSREP_PORT'] +if check_env_var("MARIADB_PORT"): + mariadb_port = int(os.environ['MARIADB_PORT']) if check_env_var("MYSQL_DBADMIN_USERNAME"): mysql_dbadmin_username = os.environ['MYSQL_DBADMIN_USERNAME'] if check_env_var("MYSQL_DBADMIN_PASSWORD"): @@ -115,7 +121,8 @@ if mysql_dbadmin_username == mysql_dbsst_username: sys.exit(1) # Set some variables for tuneables -cluster_leader_ttl = 120 +if check_env_var("CLUSTER_LEADER_TTL"): + cluster_leader_ttl = int(os.environ['CLUSTER_LEADER_TTL']) state_configmap_update_period = 10 default_sleep = 20 @@ -138,6 +145,25 @@ def ensure_state_configmap(pod_namespace, configmap_name, configmap_body): return False +def ensure_primary_service(pod_namespace, service_name, service_body): + """Ensure the primary service exists. + + Keyword arguments: + pod_namespace -- the namespace to house the service + service_name -- the service name + service_body -- the service body + """ + try: + k8s_api_instance.read_namespaced_service( + name=service_name, namespace=pod_namespace) + return True + except: + k8s_api_instance.create_namespaced_service( + namespace=pod_namespace, body=service_body) + + return False + + def run_cmd_with_logging(popenargs, logger, @@ -388,6 +414,60 @@ def set_configmap_data(key, value): return safe_update_configmap( configmap_dict=configmap_dict, configmap_patch=configmap_patch) +def safe_update_service(service_dict, service_patch): + """Update a service with locking. + + Keyword arguments: + service_dict -- a dict representing the service to be patched + service_patch -- a dict containign the patch + """ + logger.debug("Safe Patching service") + # NOTE(portdirect): Explictly set the resource version we are patching to + # ensure nothing else has modified the service since we read it. + service_patch['metadata']['resourceVersion'] = service_dict[ + 'metadata']['resource_version'] + + # Retry up to 8 times in case of 409 only. Each retry has a ~1 second + # sleep in between so do not want to exceed the roughly 10 second + # write interval per cm update. + for i in range(8): + try: + api_response = k8s_api_instance.patch_namespaced_service( + name=primary_service_name, + namespace=pod_namespace, + body=service_patch) + return True + except kubernetes.client.rest.ApiException as error: + if error.status == 409: + # This status code indicates a collision trying to write to the + # service while another instance is also trying the same. + logger.warning("Collision writing service: {0}".format(error)) + # This often happens when the replicas were started at the same + # time, and tends to be persistent. Sleep with some random + # jitter value briefly to break the synchronization. + naptime = secretsGen.uniform(0.8,1.2) + time.sleep(naptime) + else: + logger.error("Failed to set service: {0}".format(error)) + return error + logger.info("Retry writing service attempt={0} sleep={1}".format( + i+1, naptime)) + return True + +def set_primary_service_spec(key, value): + """Update a service's endpoint via patching. + + Keyword arguments: + key -- the key to be patched + value -- the value to give the key + """ + logger.debug("Setting service spec.selector key={0} to value={1}".format(key, value)) + service_dict = k8s_api_instance.read_namespaced_service( + name=primary_service_name, namespace=pod_namespace).to_dict() + service_patch = {'spec': {'selector': {}}, 'metadata': {}} + service_patch['spec']['selector'][key] = value + return safe_update_service( + service_dict=service_dict, service_patch=service_patch) def get_configmap_value(key, type='data'): """Get a configmap's key's value. @@ -469,6 +549,35 @@ def get_cluster_state(): pod_namespace=pod_namespace, configmap_name=state_configmap_name, configmap_body=initial_configmap_body) + + + initial_primary_service_body = { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": primary_service_name, + }, + "spec": { + "ports": [ + { + "name": "mysql", + "port": mariadb_port + } + ], + "selector": { + "application": "mariadb", + "component": "server", + "statefulset.kubernetes.io/pod-name": leader + } + } + } + if ensure_primary_service( + pod_namespace=pod_namespace, + service_name=primary_service_name, + service_body=initial_primary_service_body): + logger.info("Service {0} already exists".format(primary_service_name)) + else: + logger.info("Service {0} has been successfully created".format(primary_service_name)) return state @@ -480,6 +589,38 @@ def declare_myself_cluster_leader(): leader_expiry = "{0}Z".format(leader_expiry_raw.isoformat("T")) set_configmap_annotation( key='openstackhelm.openstack.org/leader.node', value=local_hostname) + logger.info("Setting primary_service's spec.selector to {0}".format(local_hostname)) + try: + set_primary_service_spec( + key='statefulset.kubernetes.io/pod-name', value=local_hostname) + except: + initial_primary_service_body = { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": primary_service_name, + }, + "spec": { + "ports": [ + { + "name": "mysql", + "port": mariadb_port + } + ], + "selector": { + "application": "mariadb", + "component": "server", + "statefulset.kubernetes.io/pod-name": local_hostname + } + } + } + if ensure_primary_service( + pod_namespace=pod_namespace, + service_name=primary_service_name, + service_body=initial_primary_service_body): + logger.info("Service {0} already exists".format(primary_service_name)) + else: + logger.info("Service {0} has been successfully created".format(primary_service_name)) set_configmap_annotation( key='openstackhelm.openstack.org/leader.expiry', value=leader_expiry) diff --git a/mariadb/templates/statefulset.yaml b/mariadb/templates/statefulset.yaml index b78f69d7c..42521f190 100644 --- a/mariadb/templates/statefulset.yaml +++ b/mariadb/templates/statefulset.yaml @@ -47,6 +47,29 @@ rules: - configmaps verbs: - create + - apiGroups: + - "" + resources: + - services + verbs: + - create + - apiGroups: + - "" + resourceNames: + - {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + resources: + - services + verbs: + - get + - patch + - apiGroups: + - "" + resourceNames: + - {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + resources: + - endpoints + verbs: + - get - apiGroups: - "" resourceNames: @@ -165,6 +188,12 @@ spec: value: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: STATE_CONFIGMAP value: {{ printf "%s-%s" .deployment_name "mariadb-state" | quote }} + - name: PRIMARY_SERVICE_NAME + value: {{ tuple "oslo_db" "primary" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: CLUSTER_LEADER_TTL + value: {{ .Values.conf.galera.cluster_leader_ttl | quote }} + - name: MARIADB_PORT + value: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} - name: MYSQL_DBADMIN_USERNAME value: {{ .Values.endpoints.oslo_db.auth.admin.username }} - name: MYSQL_DBADMIN_PASSWORD diff --git a/mariadb/values.yaml b/mariadb/values.yaml index d3bc4fb57..9daf08ab3 100644 --- a/mariadb/values.yaml +++ b/mariadb/values.yaml @@ -362,6 +362,8 @@ conf: lock_expire_after: 7200 retry_after: 3600 container_name: throttle-backups-manager + galera: + cluster_leader_ttl: 120 database: mysql_histfile: "/dev/null" my: | @@ -603,6 +605,7 @@ endpoints: direct: mariadb-server discovery: mariadb-discovery error_pages: mariadb-ingress-error-pages + primary: mariadb-server-primary host_fqdn_override: default: null path: null diff --git a/mariadb/values_overrides/primary-service.yaml b/mariadb/values_overrides/primary-service.yaml new file mode 100644 index 000000000..919dcea17 --- /dev/null +++ b/mariadb/values_overrides/primary-service.yaml @@ -0,0 +1,21 @@ +--- +manifests: + deployment_ingress: false + deployment_error: false + service_ingress: false + configmap_ingress_conf: false + configmap_ingress_etc: false + service_error: false +volume: + size: 1Gi + backup: + size: 1Gi +conf: + galera: + cluster_leader_ttl: 10 +endpoints: + oslo_db: + hosts: + default: mariadb + primary: mariadb +... diff --git a/releasenotes/notes/mariadb.yaml b/releasenotes/notes/mariadb.yaml index a045fafc6..6ab298f2f 100644 --- a/releasenotes/notes/mariadb.yaml +++ b/releasenotes/notes/mariadb.yaml @@ -55,4 +55,5 @@ mariadb: - 0.2.37 Backups verification improvements - 0.2.38 Added throttling remote backups - 0.2.39 Template changes for image 1.9 compatibility + - 0.2.40 Start.py allows to create mariadb-service-primary service and endpoint ... diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 67f2577e5..876e3dd7b 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -266,6 +266,34 @@ - ./tools/deployment/openstack-support/120-powerdns.sh - ./tools/deployment/openstack-support/130-cinder.sh +- job: + name: openstack-helm-infra-openstack-support-mariadb-service-primary + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-1node-ubuntu_focal + vars: + osh_params: + openstack_release: "2023.1" + container_distro_name: ubuntu + container_distro_version: focal + feature_gates: "ssl,primary-service" + gate_scripts: + - ./tools/deployment/openstack-support/000-prepare-k8s.sh + - ./tools/deployment/openstack-support/007-namespace-config.sh + - ./tools/deployment/openstack-support/010-ingress.sh + - ./tools/deployment/ceph/ceph.sh + - ./tools/deployment/openstack-support/025-ceph-ns-activate.sh + - ./tools/deployment/openstack-support/030-rabbitmq.sh + - ./tools/deployment/openstack-support/070-mariadb.sh + - ./tools/deployment/openstack-support/040-memcached.sh + - ./tools/deployment/openstack-support/051-libvirt-ssl.sh + - ./tools/deployment/openstack-support/060-openvswitch.sh + - ./tools/deployment/common/setup-client.sh + - ./tools/deployment/openstack-support/090-keystone.sh + - ./tools/deployment/openstack-support/100-ceph-radosgateway.sh + - ./tools/deployment/openstack-support/110-openstack-exporter.sh + - ./tools/deployment/openstack-support/120-powerdns.sh + - ./tools/deployment/openstack-support/130-cinder.sh + - job: name: openstack-helm-infra-mariadb-operator diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 0361c2cbf..9d3132b63 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -29,6 +29,7 @@ - openstack-helm-infra-openstack-support-ssl - openstack-helm-infra-metacontroller - openstack-helm-infra-mariadb-operator + - openstack-helm-infra-openstack-support-mariadb-service-primary gate: jobs: - openstack-helm-lint @@ -38,6 +39,7 @@ - openstack-helm-infra-openstack-support - openstack-helm-infra-openstack-support-rook - openstack-helm-infra-openstack-support-ssl + - openstack-helm-infra-openstack-support-mariadb-service-primary post: jobs: - publish-openstack-helm-charts