From 8a1fdb960cc0a7e206da7eccfcfec1e374888162 Mon Sep 17 00:00:00 2001 From: "IPATOV, DENIS (di0361)" Date: Thu, 19 Sep 2019 11:38:46 -0700 Subject: [PATCH] Monitoring PVC-PV-RBD and mapping them The script allows to monitor if PVC has associated PV and RBD associated with PV. usage: 001-pvc_rbd_monitoring.py [-h] [--rbd] [--pvc] [--all] [--pv] [--bin] [--config] [--debug | --silent] optional arguments: -h, --help show this help message and exit --debug, -d enable debugging (default: False) --silent, -s don't log into console (default: False) monitoring settigns: --rbd, -r check rbd (default: False) --pvc, -p check permanent volume claim (default: False) --all, -a all checks (default: False) --pv check permanent volumes (default: False) --bin, -e path to kubecl binary (default: None) --config, -c path to kubecl config (default: None) Change-Id: If88979d645c543ff778201c597f898b7c7aa0388 --- tools/fixes/pvc_pv_rbd_monitoring.py | 290 +++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 tools/fixes/pvc_pv_rbd_monitoring.py diff --git a/tools/fixes/pvc_pv_rbd_monitoring.py b/tools/fixes/pvc_pv_rbd_monitoring.py new file mode 100644 index 000000000..505c2d614 --- /dev/null +++ b/tools/fixes/pvc_pv_rbd_monitoring.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 + +import sys +import subprocess +import argparse + +import json +import logging +import logging.handlers +import os + + +# need to export path to configuration file. should work faster +if os.environ["KUBECONFIG"] !="": + CONFIG = "KUBECONFIG={}".format(os.environ["KUBECONFIG"]) +else: + CONFIG = "KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml" +if os.environ["KUBECTL"] != "": + KUBECTL = os.environ["KUBECTL"] +else: + KUBECTL = "/usr/local/bin/kubectl" + +OPTIONS = { + 'read': "-o json", + 'ns': "-n", + 'exec': "" + } + +logger = logging.getLogger(os.path.splitext(os.path.basename(sys.argv[0]))[0]) + +kubectl = "{config} {kubectl} {options} {namespace}".format(config=CONFIG, + kubectl=KUBECTL, + namespace=OPTIONS["ns"], + options=OPTIONS["read"]) + +kubectl_exec = "{config} {kubectl} {options} {namespace}".format(config=CONFIG, + kubectl=KUBECTL, + namespace=OPTIONS["ns"], + options=OPTIONS["exec"]) + + +class CustomFormatter(argparse.RawDescriptionHelpFormatter, + argparse.ArgumentDefaultsHelpFormatter): + pass + + +def parse_args(args=sys.argv[1:]): + """ parse arguments of script """ + + logger.debug("Parsing arguments") + parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__, + formatter_class=CustomFormatter) + g = parser.add_argument_group("monitoring settigns") + g.add_argument("--rbd", "-r", action="store_true", + default=False, help="check rbd") + g.add_argument("--pvc", "-p", action="store_true", + default=False, help="check permanent volume claim") + g.add_argument("--all", "-a", action="store_true", + default=False, help="all checks") + g.add_argument("--pv", action="store_true", + default=False, help="check permanent volumes") + + # configuration options + g.add_argument("--bin", "-e", action="store_const", + const=KUBECTL, + help="path to kubecl binary") + g.add_argument("--config", "-c", action="store_const", + const=CONFIG, + help="path to kubecl config") + + # debug options + g = parser.add_mutually_exclusive_group() + g.add_argument("--debug", "-d", action="store_true", + default=False, help="enable debugging") + g.add_argument("--silent", "-s", action="store_true", + default=False, help="don't log into console") + return parser.parse_args(args) + + +def setup_logging(options): + """ configure logging """ + + logger.debug("Configuring logging") + root = logging.getLogger("") + root.setLevel(logging.WARNING) + logger.setLevel(options.debug and logging.DEBUG or logging.INFO) + if not options.silent: + ch = logging.StreamHandler() + ch.setFormatter(logging.Formatter("%(levelname)s [%(name)s] %(message)s")) + root.addHandler(ch) + + +def get_podname(label, namespace): + """ return list of pods in provided namespaces with label """ + + logger.debug("Getting pod names with labels {label} in namespcae {namespace}".format(label=label, + namespace=namespace)) + command = "{command} {namespace} get pods -l {label}".format(command=kubectl, + namespace=namespace, + label=label) + logger.debug(command) + try: + output = run_command(command) + except Exception as e: + logger.error("Getting pod names for labels {label} in namespace {namespace}".format(label=label, + namespace=namespace)) + sys.exit(1) + + podnames = [] + try: + out = json.loads(output) + for item in out["items"]: + if item['kind'] == "Pod": + podnames.append(item['metadata']['name']) + except Exception as e: + sys.exit(1) + return podnames + + +def run_command(command): + """ run command in linux shell and return a result """ + + try: + result = subprocess.check_output(command, + shell=True).strip().decode('utf-8') + except Exception as e: + logger.error(e) + sys.exit(1) + return result + + +def check_rbd_status(namespace, rbd): + """ return status or rbd in json format """ + + ceph_mon = get_podname("application=ceph,component=mon", namespace)[0] + command = "{command} {namespace} exec -it {pod} -- rbd status --format json {rbd}".format(command=kubectl_exec, + namespace=namespace, + rbd=rbd, pod=ceph_mon) + out = run_command(command) + try: + output = json.loads(out) + except Exception as e: + output = "{{'rbd': {}, 'status': {} }}".format(rbd, e) + return output + + +def get_pvc(namespace): + """ return list of permanent volume claim in specific namespace """ + + command = "{command} {namespace} get pvc".format(command=kubectl, + namespace=namespace) + out = run_command(command) + try: + output = json.loads(out) + except Exception as e: + # output = {'namespace': namespace, 'status': e} + output = [] + return output + + +def get_pv(persistentvolume, namespace): + """ return list of persistent volumes in specific namespace """ + + command = "{command} {namespace} get pv {pv}".format(command=kubectl, + namespace=namespace, + pv=persistentvolume) + out = run_command(command) + try: + output = json.loads(out) + except Exception as e: + output = {'persistentvolume': namespace, 'status': e} + return output + + +def get_namespaces(): + """ return list of namespaces """ + + command = "{command} {namespace} get ns -o json".format(command=kubectl_exec, + namespace="ceph") + out = run_command(command) + try: + output = json.loads(out) + except Exception as e: + output = {"Error": e} + return output + + +def get_rbd_list(namespace): + """ return list of rbd in json format """ + + ceph_mon = get_podname("application=ceph,component=mon", namespace)[0] + command = "{command} {namespace} exec -it {pod} -- rbd ls --format json".format(command=kubectl_exec, + namespace=namespace, + pod=ceph_mon) + out = run_command(command) + logger.debug(out) + try: + output = json.loads(out) + except Exception as e: + output = "{{'namaspace': {}, 'status': {} }}".format(namespace, e) + logger.error("parse json rbd list output: {}".format(e)) + return output + + +def monitoring_pvc(): + logger.info("PVCs aren't associated with RBD") + rbds = get_rbd_list("ceph") + logger.info("Gettign list of namespaces") + namespaces = get_namespaces() + for namespace in namespaces['items']: + ns = namespace['metadata']['name'] + logger.info("Checking namespace: {}".format(ns)) + pvc = get_pvc(ns) + for pv in pvc['items']: + rbd = get_pv(pv['spec']['volumeName'], ns) + rbd = rbd['spec']['rbd']['image'] + if rbd not in rbds: + print ("pvc_doesnot_have_rbd:{{namespace={},name={}}} 0".format(ns, pv['metadata']['name'])) + + +def monitoring_rbd(): + logger.info("RBD volumes aren't associated with PVC") + r = get_rbd_list("ceph") + rbds = [] + namespaces = get_namespaces() + for namespace in namespaces['items']: + ns = namespace['metadata']['name'] + pvc = get_pvc(ns) + for pv in pvc['items']: + rbd = get_pv(pv['spec']['volumeName'], ns) + rbd = rbd['spec']['rbd']['image'] + rbds.append(rbd) + logger.info(rbd) + for i in r: + if i not in rbds: + logger.debug(i) + print ("rbd_doesnot_have_pvc:{{name={}}} {}".format(i, len(check_rbd_status("ceph", i)['watchers']))) + + +def monitoring_pv(): + logger.info("PVs aren't associated with PVC ") + command = "{config} {kubectl} get pv -o json".format(config=CONFIG, kubectl=KUBECTL) + logger.debug(command) + out = run_command(command) + logger.debug(out) + try: + output = json.loads(out) + for pv in output['items']: + if pv['status']['phase'] == "Released": + print ("pv_released:{{name={},status={}}} 0".format(pv['metadata']['name'], pv['status']['phase'])) + except Exception as e: + logger.error("gettign rbd list: {}".format(e)) + sys.exit(1) + + +def monitoring(): + monitoring_rbd() + monitoring_pvc() + monitoring_pv() + + +def main(): + # nagios monitoring + # 0 - Service is OK. + # 1 - Service has a WARNING. + # 2 - Service is in a CRITICAL status. + # 3 - Service status is UNKNOWN + + # promotheus text file + options = parse_args() + setup_logging(options) + + + if options.all: + logger.info("all") + monitoring() + if options.rbd: + logger.info("rbd") + monitoring_rbd() + if options.pvc: + logger.info("pvc") + monitoring_pvc() + if options.pv: + logger.info("pv") + monitoring_pv() + sys.exit(1) + + +if __name__ == "__main__": + main()