Mysqlclient utility enhancement

In this patchset, mysqlclient utility is enhanced with below changes: 1) Added kubectl to mysqlclient-utility image and rootwrap to enable the utility container to invoke kubectl. 2) Added implementation for dbutils tool to perform manual local and remote backup/restore of Mariadb databases 3) Added helper functions to retrive database and database table data from local and remove database archives 4) Added mysql on demand job to aid the backup and restore 5) Added access cli to retrieve database and database table data from live Mariadb databases 6) Added mysql prompt implementation to the dbutils tool to provide easy access of mariadb in different namespaces 7) Added test database functions to support automated Mariadb backup/restore testing Change-Id: I51a8bf4131f2369520240d32d6fc81ed56160d43
2020-05-18 21:33:01 +00:00
parent 2b73c87c48
commit 1ac9f7ae2a
6 changed files with 423 additions and 3 deletions
--- a/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl
+++ b/charts/mysqlclient-utility/templates/bin/utility/_mariadb_ondemand_job.sh.tpl
@@ -0,0 +1,178 @@
+#!/bin/bash
+
+{{- $envAll := . }}
+
+export MARIADB_POD_NAMESPACE=$1
+if [[ $MARIADB_POD_NAMESPACE == "" ]]; then
+  echo "No namespace given - cannot spawn ondemand job."
+  exit 1
+fi
+
+export MARIADB_RGW_SECRET={{ $envAll.Values.conf.mariadb_backup_restore.secrets.rgw_secret }}
+export MARIADB_CONF_SECRET={{ $envAll.Values.conf.mariadb_backup_restore.secrets.conf_secret }}
+export MARIADB_IMAGE_NAME=$(kubectl get cronjob -n ${MARIADB_POD_NAMESPACE} mariadb-backup -o yaml -o jsonpath="{range .spec.jobTemplate.spec.template.spec.containers[*]}{.image}{'\n'}{end}" | grep mysqlclient-utility)
+export MARIADB_BACKUP_BASE_PATH=$(kubectl get secret -o yaml -n ${MARIADB_POD_NAMESPACE} ${MARIADB_CONF_SECRET} | grep BACKUP_BASE_PATH | awk '{print $2}' | base64 -d)
+
+if [[ $MARIADB_IMAGE_NAME == "" ]]; then
+  echo "Cannot find the utility image for populating MARIADB_IMAGE_NAME variable."
+  exit 1
+fi
+
+cat <<EOF | kubectl create -n $MARIADB_POD_NAMESPACE -f -
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mariadb-ondemand
+  annotations:
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
+  labels:
+{{ tuple $envAll "mariadb-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+{{ tuple $envAll "mariadb-ondemand" "ondemand" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
+    spec:
+{{ dict "envAll" $envAll "application" "mariadb_ondemand" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }}
+      restartPolicy: OnFailure
+      nodeSelector:
+        {{ .Values.labels.utility.node_selector_key }}: {{ .Values.labels.utility.node_selector_value }}
+      initContainers:
+        - name: ondemand-perms
+          image: ${MARIADB_IMAGE_NAME}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "ondemand_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - chown
+            - -R
+            - "65534:65534"
+            - ${MARIADB_BACKUP_BASE_PATH}
+          volumeMounts:
+            - mountPath: /tmp
+              name: pod-tmp
+            - mountPath: ${MARIADB_BACKUP_BASE_PATH}
+              name: mariadb-backup-dir
+      containers:
+        - name: mariadb-ondemand
+          image: ${MARIADB_IMAGE_NAME}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_ondemand | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+{{ dict "envAll" $envAll "application" "mariadb_ondemand" "container" "mariadb_ondemand" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
+          command:
+            - /bin/sleep
+            - "1000000"
+          env:
+            - name: MARIADB_BACKUP_BASE_DIR
+              valueFrom:
+                secretKeyRef:
+                  key: BACKUP_BASE_PATH
+                  name: ${MARIADB_CONF_SECRET}
+            - name: MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP
+              valueFrom:
+                secretKeyRef:
+                  key: LOCAL_DAYS_TO_KEEP
+                  name: ${MARIADB_CONF_SECRET}
+            - name: REMOTE_BACKUP_ENABLED
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_ENABLED
+                  name: ${MARIADB_CONF_SECRET}
+            - name: MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_DAYS_TO_KEEP
+                  name: ${MARIADB_CONF_SECRET}
+            - name: MARIADB_POD_NAMESPACE
+              value: ${MARIADB_POD_NAMESPACE}
+            - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS
+              valueFrom:
+                secretKeyRef:
+                  key: MYSQLDUMP_OPTIONS
+                  name: ${MARIADB_CONF_SECRET}
+            - name: STORAGE_POLICY
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_STORAGE_POLICY
+                  name: ${MARIADB_CONF_SECRET}
+            - name: CONTAINER_NAME
+              valueFrom:
+                secretKeyRef:
+                  key: REMOTE_BACKUP_CONTAINER
+                  name: ${MARIADB_CONF_SECRET}
+            - name: OS_IDENTITY_API_VERSION
+              value: "3"
+            - name: OS_AUTH_URL
+              valueFrom:
+                secretKeyRef:
+                  name: ${MARIADB_RGW_SECRET}
+                  key: OS_AUTH_URL
+            - name: OS_REGION_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${MARIADB_RGW_SECRET}
+                  key: OS_REGION_NAME
+            - name: OS_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${MARIADB_RGW_SECRET}
+                  key: OS_USERNAME
+            - name: OS_PROJECT_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${MARIADB_RGW_SECRET}
+                  key: OS_PROJECT_NAME
+            - name: OS_USER_DOMAIN_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${MARIADB_RGW_SECRET}
+                  key: OS_USER_DOMAIN_NAME
+            - name: OS_PROJECT_DOMAIN_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: ${MARIADB_RGW_SECRET}
+                  key: OS_PROJECT_DOMAIN_NAME
+            - name: OS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: ${MARIADB_RGW_SECRET}
+                  key: OS_PASSWORD
+          volumeMounts:
+            - name: pod-tmp
+              mountPath: /tmp
+            - mountPath: /tmp/restore_mariadb.sh
+              name: mariadb-bin
+              readOnly: true
+              subPath: restore_mariadb.sh
+            - mountPath: /tmp/restore_main.sh
+              name: mariadb-bin
+              readOnly: true
+              subPath: restore_main.sh
+            - mountPath: /tmp/backup_mariadb.sh
+              name: mariadb-bin
+              readOnly: true
+              subPath: backup_mariadb.sh
+            - mountPath: /tmp/backup_main.sh
+              name: mariadb-bin
+              readOnly: true
+              subPath: backup_main.sh
+            - mountPath: ${MARIADB_BACKUP_BASE_PATH}
+              name: mariadb-backup-dir
+            - name: mariadb-secrets
+              mountPath: /etc/mysql/admin_user.cnf
+              subPath: admin_user.cnf
+              readOnly: true
+      volumes:
+        - name: pod-tmp
+          emptyDir: {}
+        - name: mariadb-secrets
+          secret:
+            secretName: mariadb-secrets
+            defaultMode: 292
+        - name: mariadb-bin
+          configMap:
+            name: mariadb-bin
+            defaultMode: 365
+        - name: mariadb-backup-dir
+          persistentVolumeClaim:
+            claimName: mariadb-backup-data
+EOF
--- a/charts/mysqlclient-utility/templates/bin/utility/_mysqlutils.sh.tpl
+++ b/charts/mysqlclient-utility/templates/bin/utility/_mysqlutils.sh.tpl
@@ -0,0 +1,159 @@
+#!/bin/bash
+
+function database_cmd() {
+  echo "mysql --defaults-file=/etc/mysql/admin_user.cnf --connect-timeout 10"
+}
+
+# Params: <namespace> <pod_name>
+function show_databases() {
+
+  SHOW_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="SHOW DATABASES;"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${SHOW_ARGS[1]}" "${SHOW_ARGS[2]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <database> <pod_name>
+function show_tables() {
+
+  SHOW_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="USE ${SHOW_ARGS[2]};SHOW TABLES;"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${SHOW_ARGS[1]}" "${SHOW_ARGS[3]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <database> <table> <pod_name>
+function show_rows() {
+
+  SHOW_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="USE ${SHOW_ARGS[2]};SELECT * FROM ${SHOW_ARGS[3]};"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${SHOW_ARGS[1]}" "${SHOW_ARGS[4]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <database> <table> <pod_name>
+function show_schema() {
+
+  SHOW_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="USE ${SHOW_ARGS[2]};DESC ${SHOW_ARGS[3]};"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${SHOW_ARGS[1]}" "${SHOW_ARGS[4]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <pod_name>
+function sql_prompt() {
+
+  SHOW_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${SHOW_ARGS[1]}" "${SHOW_ARGS[2]}" -- ${MYSQL_CMD}
+}
+
+# Params: <namespace> <database> <pod_name>
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function create_database() {
+
+  CREATE_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="CREATE DATABASE test_${CREATE_ARGS[2]};"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${CREATE_ARGS[1]}" "${CREATE_ARGS[3]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <database> <table> <pod_name>
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function create_table() {
+
+  CREATE_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="USE test_${CREATE_ARGS[2]};CREATE TABLE ${CREATE_ARGS[3]} \
+    ( id int(11) NOT NULL AUTO_INCREMENT, name varchar(255) NOT NULL, user_id int(11) DEFAULT 0, PRIMARY KEY (id) );"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${CREATE_ARGS[1]}" "${CREATE_ARGS[4]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <database> <table> <pod_name>
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function create_row() {
+
+  CREATE_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="USE test_${CREATE_ARGS[2]};LOCK TABLES ${CREATE_ARGS[3]} WRITE \
+    ;INSERT INTO ${CREATE_ARGS[3]} (name) value ('name') \
+    ;UPDATE ${CREATE_ARGS[3]} SET user_id=id,name=CONCAT(name,user_id) WHERE id = LAST_INSERT_ID() \
+    ;UNLOCK TABLES;"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${CREATE_ARGS[1]}" "${CREATE_ARGS[4]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <database> <table> <colname> <value> <pod_name>
+#   Where: <colname> = <value> is the condition used to find the row to be deleted.
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function delete_row() {
+
+  DELETE_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="USE test_${DELETE_ARGS[2]};DELETE FROM ${DELETE_ARGS[3]} WHERE ${DELETE_ARGS[4]} = '${DELETE_ARGS[5]}';"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${DELETE_ARGS[1]}" "${DELETE_ARGS[6]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <database> <tablename> <pod_name>
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function delete_table() {
+
+  DELETE_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="USE test_${DELETE_ARGS[2]};DROP TABLE IF EXISTS ${DELETE_ARGS[3]};"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${DELETE_ARGS[1]}" "${DELETE_ARGS[4]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
+
+# Params: <namespace> <database> <pod_name>
+#   NOTE: "test_" is automatically prepended before the provided database
+#         name, in order to prevent accidental modification/deletion of
+#         an application database.
+function delete_database() {
+
+  DELETE_ARGS=("$@")
+
+  MYSQL_CMD=$(database_cmd)
+  DB_ARGS="DROP DATABASE IF EXISTS test_${DELETE_ARGS[2]};"
+
+  # Execute the command in the on-demand pod
+  kubectl exec -it -n "${DELETE_ARGS[1]}" "${DELETE_ARGS[3]}" -- ${MYSQL_CMD} --execute="$DB_ARGS"
+}
--- a/charts/mysqlclient-utility/templates/configmap-bin.yaml
+++ b/charts/mysqlclient-utility/templates/configmap-bin.yaml
@@ -40,6 +40,15 @@ data:
  utilscli: |
 {{ tuple "bin/utility/_utilscli.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}

+  mariadb-ondemand-job.sh: |
+{{ tuple "bin/utility/_mariadb_ondemand_job.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+  dbutils: |
+{{ tuple "bin/utility/_dbutils.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
+  mysqlutils.sh: |
+{{ tuple "bin/utility/_mysqlutils.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
+
  bootstrap.sh: |
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

--- a/charts/mysqlclient-utility/templates/deployment-utility.yaml
+++ b/charts/mysqlclient-utility/templates/deployment-utility.yaml
@@ -33,6 +33,33 @@ rules:
      - watch
      - exec
      - create
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "batch"
+    resources:
+      - cronjobs
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "batch"
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1beta1
 kind: ClusterRoleBinding
@@ -93,6 +120,11 @@ spec:
              - utilscli
            initialDelaySeconds: 5
            periodSeconds: 15
+          env:
+            - name: BACKUP_RESTORE_NAMESPACE_LIST
+              value: {{ .Values.conf.mariadb_backup_restore.enabled_namespaces | quote }}
+            - name: BACKUP_RESTORE_SCOPE
+              value: "mariadb"
          volumeMounts:
            - name: mysqlclient-utility-bin-utilscli
              mountPath: /tmp/bootstrap.sh
@@ -110,6 +142,18 @@ spec:
              mountPath: /usr/local/bin/mysqlclient-utility-rootwrap
              subPath: mysqlclient-utility-rootwrap
              readOnly: true
+            - name: mysqlclient-utility-bin-utilscli
+              mountPath: /usr/local/bin/dbutils
+              subPath: dbutils
+              readOnly: true
+            - name: mysqlclient-utility-bin-utilscli
+              mountPath: /tmp/mysqlutils.sh
+              subPath: mysqlutils.sh
+              readOnly: true
+            - name: mysqlclient-utility-bin-utilscli
+              mountPath: /tmp/mariadb-ondemand-job.sh
+              subPath: mariadb-ondemand-job.sh
+              readOnly: true
            - name: mysqlclient-utility-sudoers
              mountPath: /etc/sudoers.d/utilscli-sudo
              subPath: utilscli-sudo
--- a/charts/mysqlclient-utility/values.yaml
+++ b/charts/mysqlclient-utility/values.yaml
@@ -42,6 +42,17 @@ pod:
        mysqlclient_utility:
          allowPrivilegeEscalation: true
          readOnlyRootFilesystem: false
+    mariadb_ondemand:
+      pod:
+        runAsUser: 65534
+      container:
+        ondemand_perms:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
+        mariadb_ondemand:
+          runAsUser: 65534
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
  dns_policy: "ClusterFirstWithHostNet"
  replicas:
    utility: 1
@@ -79,9 +90,20 @@ pod:
        limits:
          memory: "1024Mi"
          cpu: "2000m"
-
+      mariadb_ondemand:
+        requests:
+          memory: "128Mi"
+          cpu: "100m"
+        limits:
+          memory: "1024Mi"
+          cpu: "2000m"

 conf:
+  mariadb_backup_restore:
+    enabled_namespaces: ""
+    secrets:
+      rgw_secret: mariadb-backup-user
+      conf_secret: mariadb-backup-restore
  features:
    utility: true
  mysqlclientconf:
@@ -91,7 +113,8 @@ conf:
      # mysqlclient-rootwrap command filters for mysqlclient utility container
      # This file should be owned by (and only-writeable by) the root user
      mysql: CommandFilter, mysql, root
-      #kubectl: CommandFilter, kubectl, root
+      kubectl: CommandFilter, kubectl, root
+      dbutils: CommandFilter, dbutils, nobody
  mysqlclientrootwrapconf:
    DEFAULT:
      # Configuration for mysqlclient-rootwrap
@@ -164,4 +187,4 @@ manifests:
  configmap_bin: true
  configmap_etc_client: true
  configmap_etc_sudoers: true
-  deployment_utility: true
+  deployment_utility: true
--- a/images/mysqlclient-utility/Dockerfile.ubuntu_bionic
+++ b/images/mysqlclient-utility/Dockerfile.ubuntu_bionic
@@ -8,6 +8,8 @@ LABEL org.opencontainers.image.authors='airship-discuss@lists.airshipit.org, irc
      org.opencontainers.image.vendor='The Airship Authors' \
      org.opencontainers.image.licenses='Apache-2.0'

+ARG KUBE_VERSION=1.17.3
+
 RUN set -xe \
    && export DEBIAN_FRONTEND=noninteractive \
    && sed -i '/nobody/d' /etc/passwd \
@@ -29,7 +31,12 @@ RUN set -xe \
         python-swiftclient \
    && sed -i 's/$PrivDropToUser syslog/$PrivDropToUser nobody/' /etc/rsyslog.conf \
    && apt-get clean -y \
+    && TMP_DIR=$(mktemp --directory) && cd ${TMP_DIR} \
+    && curl -sSL https://dl.k8s.io/v${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
+    && mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
+    && chmod +x /usr/bin/kubectl \
    && rm -rf \
+      ${TMP_DIR} \
      /tmp/* \
      /var/cache/debconf/* \
      /var/lib/apt/lists/* \