Monitoring PVC-PV-RBD and mapping them
The script allows to monitor if PVC has associated PV and RBD associated with PV.
usage: 001-pvc_rbd_monitoring.py [-h] [--rbd] [--pvc] [--all] [--pv] [--bin]
[--config] [--debug | --silent]
optional arguments:
-h, --help show this help message and exit
--debug, -d enable debugging (default: False)
--silent, -s don't log into console (default: False)
monitoring settigns:
--rbd, -r check rbd (default: False)
--pvc, -p check permanent volume claim (default: False)
--all, -a all checks (default: False)
--pv check permanent volumes (default: False)
--bin, -e path to kubecl binary (default: None)
--config, -c path to kubecl config (default: None)
Change-Id: If88979d645c543ff778201c597f898b7c7aa0388
This commit is contained in:
IPATOV, DENIS (di0361)
Kaspars Skels
parent
064185e1af
commit
8a1fdb960c
290
tools/fixes/pvc_pv_rbd_monitoring.py
Normal file
290
tools/fixes/pvc_pv_rbd_monitoring.py
Normal file
@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import subprocess
|
||||
import argparse
|
||||
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
|
||||
|
||||
# need to export path to configuration file. should work faster
|
||||
if os.environ["KUBECONFIG"] !="":
|
||||
CONFIG = "KUBECONFIG={}".format(os.environ["KUBECONFIG"])
|
||||
else:
|
||||
CONFIG = "KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml"
|
||||
if os.environ["KUBECTL"] != "":
|
||||
KUBECTL = os.environ["KUBECTL"]
|
||||
else:
|
||||
KUBECTL = "/usr/local/bin/kubectl"
|
||||
|
||||
OPTIONS = {
|
||||
'read': "-o json",
|
||||
'ns': "-n",
|
||||
'exec': ""
|
||||
}
|
||||
|
||||
logger = logging.getLogger(os.path.splitext(os.path.basename(sys.argv[0]))[0])
|
||||
|
||||
kubectl = "{config} {kubectl} {options} {namespace}".format(config=CONFIG,
|
||||
kubectl=KUBECTL,
|
||||
namespace=OPTIONS["ns"],
|
||||
options=OPTIONS["read"])
|
||||
|
||||
kubectl_exec = "{config} {kubectl} {options} {namespace}".format(config=CONFIG,
|
||||
kubectl=KUBECTL,
|
||||
namespace=OPTIONS["ns"],
|
||||
options=OPTIONS["exec"])
|
||||
|
||||
|
||||
class CustomFormatter(argparse.RawDescriptionHelpFormatter,
|
||||
argparse.ArgumentDefaultsHelpFormatter):
|
||||
pass
|
||||
|
||||
|
||||
def parse_args(args=sys.argv[1:]):
|
||||
""" parse arguments of script """
|
||||
|
||||
logger.debug("Parsing arguments")
|
||||
parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__,
|
||||
formatter_class=CustomFormatter)
|
||||
g = parser.add_argument_group("monitoring settigns")
|
||||
g.add_argument("--rbd", "-r", action="store_true",
|
||||
default=False, help="check rbd")
|
||||
g.add_argument("--pvc", "-p", action="store_true",
|
||||
default=False, help="check permanent volume claim")
|
||||
g.add_argument("--all", "-a", action="store_true",
|
||||
default=False, help="all checks")
|
||||
g.add_argument("--pv", action="store_true",
|
||||
default=False, help="check permanent volumes")
|
||||
|
||||
# configuration options
|
||||
g.add_argument("--bin", "-e", action="store_const",
|
||||
const=KUBECTL,
|
||||
help="path to kubecl binary")
|
||||
g.add_argument("--config", "-c", action="store_const",
|
||||
const=CONFIG,
|
||||
help="path to kubecl config")
|
||||
|
||||
# debug options
|
||||
g = parser.add_mutually_exclusive_group()
|
||||
g.add_argument("--debug", "-d", action="store_true",
|
||||
default=False, help="enable debugging")
|
||||
g.add_argument("--silent", "-s", action="store_true",
|
||||
default=False, help="don't log into console")
|
||||
return parser.parse_args(args)
|
||||
|
||||
|
||||
def setup_logging(options):
|
||||
""" configure logging """
|
||||
|
||||
logger.debug("Configuring logging")
|
||||
root = logging.getLogger("")
|
||||
root.setLevel(logging.WARNING)
|
||||
logger.setLevel(options.debug and logging.DEBUG or logging.INFO)
|
||||
if not options.silent:
|
||||
ch = logging.StreamHandler()
|
||||
ch.setFormatter(logging.Formatter("%(levelname)s [%(name)s] %(message)s"))
|
||||
root.addHandler(ch)
|
||||
|
||||
|
||||
def get_podname(label, namespace):
|
||||
""" return list of pods in provided namespaces with label """
|
||||
|
||||
logger.debug("Getting pod names with labels {label} in namespcae {namespace}".format(label=label,
|
||||
namespace=namespace))
|
||||
command = "{command} {namespace} get pods -l {label}".format(command=kubectl,
|
||||
namespace=namespace,
|
||||
label=label)
|
||||
logger.debug(command)
|
||||
try:
|
||||
output = run_command(command)
|
||||
except Exception as e:
|
||||
logger.error("Getting pod names for labels {label} in namespace {namespace}".format(label=label,
|
||||
namespace=namespace))
|
||||
sys.exit(1)
|
||||
|
||||
podnames = []
|
||||
try:
|
||||
out = json.loads(output)
|
||||
for item in out["items"]:
|
||||
if item['kind'] == "Pod":
|
||||
podnames.append(item['metadata']['name'])
|
||||
except Exception as e:
|
||||
sys.exit(1)
|
||||
return podnames
|
||||
|
||||
|
||||
def run_command(command):
|
||||
""" run command in linux shell and return a result """
|
||||
|
||||
try:
|
||||
result = subprocess.check_output(command,
|
||||
shell=True).strip().decode('utf-8')
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
sys.exit(1)
|
||||
return result
|
||||
|
||||
|
||||
def check_rbd_status(namespace, rbd):
|
||||
""" return status or rbd in json format """
|
||||
|
||||
ceph_mon = get_podname("application=ceph,component=mon", namespace)[0]
|
||||
command = "{command} {namespace} exec -it {pod} -- rbd status --format json {rbd}".format(command=kubectl_exec,
|
||||
namespace=namespace,
|
||||
rbd=rbd, pod=ceph_mon)
|
||||
out = run_command(command)
|
||||
try:
|
||||
output = json.loads(out)
|
||||
except Exception as e:
|
||||
output = "{{'rbd': {}, 'status': {} }}".format(rbd, e)
|
||||
return output
|
||||
|
||||
|
||||
def get_pvc(namespace):
|
||||
""" return list of permanent volume claim in specific namespace """
|
||||
|
||||
command = "{command} {namespace} get pvc".format(command=kubectl,
|
||||
namespace=namespace)
|
||||
out = run_command(command)
|
||||
try:
|
||||
output = json.loads(out)
|
||||
except Exception as e:
|
||||
# output = {'namespace': namespace, 'status': e}
|
||||
output = []
|
||||
return output
|
||||
|
||||
|
||||
def get_pv(persistentvolume, namespace):
|
||||
""" return list of persistent volumes in specific namespace """
|
||||
|
||||
command = "{command} {namespace} get pv {pv}".format(command=kubectl,
|
||||
namespace=namespace,
|
||||
pv=persistentvolume)
|
||||
out = run_command(command)
|
||||
try:
|
||||
output = json.loads(out)
|
||||
except Exception as e:
|
||||
output = {'persistentvolume': namespace, 'status': e}
|
||||
return output
|
||||
|
||||
|
||||
def get_namespaces():
|
||||
""" return list of namespaces """
|
||||
|
||||
command = "{command} {namespace} get ns -o json".format(command=kubectl_exec,
|
||||
namespace="ceph")
|
||||
out = run_command(command)
|
||||
try:
|
||||
output = json.loads(out)
|
||||
except Exception as e:
|
||||
output = {"Error": e}
|
||||
return output
|
||||
|
||||
|
||||
def get_rbd_list(namespace):
|
||||
""" return list of rbd in json format """
|
||||
|
||||
ceph_mon = get_podname("application=ceph,component=mon", namespace)[0]
|
||||
command = "{command} {namespace} exec -it {pod} -- rbd ls --format json".format(command=kubectl_exec,
|
||||
namespace=namespace,
|
||||
pod=ceph_mon)
|
||||
out = run_command(command)
|
||||
logger.debug(out)
|
||||
try:
|
||||
output = json.loads(out)
|
||||
except Exception as e:
|
||||
output = "{{'namaspace': {}, 'status': {} }}".format(namespace, e)
|
||||
logger.error("parse json rbd list output: {}".format(e))
|
||||
return output
|
||||
|
||||
|
||||
def monitoring_pvc():
|
||||
logger.info("PVCs aren't associated with RBD")
|
||||
rbds = get_rbd_list("ceph")
|
||||
logger.info("Gettign list of namespaces")
|
||||
namespaces = get_namespaces()
|
||||
for namespace in namespaces['items']:
|
||||
ns = namespace['metadata']['name']
|
||||
logger.info("Checking namespace: {}".format(ns))
|
||||
pvc = get_pvc(ns)
|
||||
for pv in pvc['items']:
|
||||
rbd = get_pv(pv['spec']['volumeName'], ns)
|
||||
rbd = rbd['spec']['rbd']['image']
|
||||
if rbd not in rbds:
|
||||
print ("pvc_doesnot_have_rbd:{{namespace={},name={}}} 0".format(ns, pv['metadata']['name']))
|
||||
|
||||
|
||||
def monitoring_rbd():
|
||||
logger.info("RBD volumes aren't associated with PVC")
|
||||
r = get_rbd_list("ceph")
|
||||
rbds = []
|
||||
namespaces = get_namespaces()
|
||||
for namespace in namespaces['items']:
|
||||
ns = namespace['metadata']['name']
|
||||
pvc = get_pvc(ns)
|
||||
for pv in pvc['items']:
|
||||
rbd = get_pv(pv['spec']['volumeName'], ns)
|
||||
rbd = rbd['spec']['rbd']['image']
|
||||
rbds.append(rbd)
|
||||
logger.info(rbd)
|
||||
for i in r:
|
||||
if i not in rbds:
|
||||
logger.debug(i)
|
||||
print ("rbd_doesnot_have_pvc:{{name={}}} {}".format(i, len(check_rbd_status("ceph", i)['watchers'])))
|
||||
|
||||
|
||||
def monitoring_pv():
|
||||
logger.info("PVs aren't associated with PVC ")
|
||||
command = "{config} {kubectl} get pv -o json".format(config=CONFIG, kubectl=KUBECTL)
|
||||
logger.debug(command)
|
||||
out = run_command(command)
|
||||
logger.debug(out)
|
||||
try:
|
||||
output = json.loads(out)
|
||||
for pv in output['items']:
|
||||
if pv['status']['phase'] == "Released":
|
||||
print ("pv_released:{{name={},status={}}} 0".format(pv['metadata']['name'], pv['status']['phase']))
|
||||
except Exception as e:
|
||||
logger.error("gettign rbd list: {}".format(e))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def monitoring():
|
||||
monitoring_rbd()
|
||||
monitoring_pvc()
|
||||
monitoring_pv()
|
||||
|
||||
|
||||
def main():
|
||||
# nagios monitoring
|
||||
# 0 - Service is OK.
|
||||
# 1 - Service has a WARNING.
|
||||
# 2 - Service is in a CRITICAL status.
|
||||
# 3 - Service status is UNKNOWN
|
||||
|
||||
# promotheus text file
|
||||
options = parse_args()
|
||||
setup_logging(options)
|
||||
|
||||
|
||||
if options.all:
|
||||
logger.info("all")
|
||||
monitoring()
|
||||
if options.rbd:
|
||||
logger.info("rbd")
|
||||
monitoring_rbd()
|
||||
if options.pvc:
|
||||
logger.info("pvc")
|
||||
monitoring_pvc()
|
||||
if options.pv:
|
||||
logger.info("pv")
|
||||
monitoring_pv()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user