Reference Airship manifests, CICD, and reference architecture.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

291 lines
9.8KB

  1. #!/usr/bin/env python3
  2. import sys
  3. import subprocess
  4. import argparse
  5. import json
  6. import logging
  7. import logging.handlers
  8. import os
  9. # need to export path to configuration file. should work faster
  10. if os.environ["KUBECONFIG"] !="":
  11. CONFIG = "KUBECONFIG={}".format(os.environ["KUBECONFIG"])
  12. else:
  13. CONFIG = "KUBECONFIG=/etc/kubernetes/admin/kubeconfig.yaml"
  14. if os.environ["KUBECTL"] != "":
  15. KUBECTL = os.environ["KUBECTL"]
  16. else:
  17. KUBECTL = "/usr/local/bin/kubectl"
  18. OPTIONS = {
  19. 'read': "-o json",
  20. 'ns': "-n",
  21. 'exec': ""
  22. }
  23. logger = logging.getLogger(os.path.splitext(os.path.basename(sys.argv[0]))[0])
  24. kubectl = "{config} {kubectl} {options} {namespace}".format(config=CONFIG,
  25. kubectl=KUBECTL,
  26. namespace=OPTIONS["ns"],
  27. options=OPTIONS["read"])
  28. kubectl_exec = "{config} {kubectl} {options} {namespace}".format(config=CONFIG,
  29. kubectl=KUBECTL,
  30. namespace=OPTIONS["ns"],
  31. options=OPTIONS["exec"])
  32. class CustomFormatter(argparse.RawDescriptionHelpFormatter,
  33. argparse.ArgumentDefaultsHelpFormatter):
  34. pass
  35. def parse_args(args=sys.argv[1:]):
  36. """ parse arguments of script """
  37. logger.debug("Parsing arguments")
  38. parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__,
  39. formatter_class=CustomFormatter)
  40. g = parser.add_argument_group("monitoring settigns")
  41. g.add_argument("--rbd", "-r", action="store_true",
  42. default=False, help="check rbd")
  43. g.add_argument("--pvc", "-p", action="store_true",
  44. default=False, help="check permanent volume claim")
  45. g.add_argument("--all", "-a", action="store_true",
  46. default=False, help="all checks")
  47. g.add_argument("--pv", action="store_true",
  48. default=False, help="check permanent volumes")
  49. # configuration options
  50. g.add_argument("--bin", "-e", action="store_const",
  51. const=KUBECTL,
  52. help="path to kubecl binary")
  53. g.add_argument("--config", "-c", action="store_const",
  54. const=CONFIG,
  55. help="path to kubecl config")
  56. # debug options
  57. g = parser.add_mutually_exclusive_group()
  58. g.add_argument("--debug", "-d", action="store_true",
  59. default=False, help="enable debugging")
  60. g.add_argument("--silent", "-s", action="store_true",
  61. default=False, help="don't log into console")
  62. return parser.parse_args(args)
  63. def setup_logging(options):
  64. """ configure logging """
  65. logger.debug("Configuring logging")
  66. root = logging.getLogger("")
  67. root.setLevel(logging.WARNING)
  68. logger.setLevel(options.debug and logging.DEBUG or logging.INFO)
  69. if not options.silent:
  70. ch = logging.StreamHandler()
  71. ch.setFormatter(logging.Formatter("%(levelname)s [%(name)s] %(message)s"))
  72. root.addHandler(ch)
  73. def get_podname(label, namespace):
  74. """ return list of pods in provided namespaces with label """
  75. logger.debug("Getting pod names with labels {label} in namespcae {namespace}".format(label=label,
  76. namespace=namespace))
  77. command = "{command} {namespace} get pods -l {label}".format(command=kubectl,
  78. namespace=namespace,
  79. label=label)
  80. logger.debug(command)
  81. try:
  82. output = run_command(command)
  83. except Exception as e:
  84. logger.error("Getting pod names for labels {label} in namespace {namespace}".format(label=label,
  85. namespace=namespace))
  86. sys.exit(1)
  87. podnames = []
  88. try:
  89. out = json.loads(output)
  90. for item in out["items"]:
  91. if item['kind'] == "Pod":
  92. podnames.append(item['metadata']['name'])
  93. except Exception as e:
  94. sys.exit(1)
  95. return podnames
  96. def run_command(command):
  97. """ run command in linux shell and return a result """
  98. try:
  99. result = subprocess.check_output(command,
  100. shell=True).strip().decode('utf-8')
  101. except Exception as e:
  102. logger.error(e)
  103. sys.exit(1)
  104. return result
  105. def check_rbd_status(namespace, rbd):
  106. """ return status or rbd in json format """
  107. ceph_mon = get_podname("application=ceph,component=mon", namespace)[0]
  108. command = "{command} {namespace} exec -it {pod} -- rbd status --format json {rbd}".format(command=kubectl_exec,
  109. namespace=namespace,
  110. rbd=rbd, pod=ceph_mon)
  111. out = run_command(command)
  112. try:
  113. output = json.loads(out)
  114. except Exception as e:
  115. output = "{{'rbd': {}, 'status': {} }}".format(rbd, e)
  116. return output
  117. def get_pvc(namespace):
  118. """ return list of permanent volume claim in specific namespace """
  119. command = "{command} {namespace} get pvc".format(command=kubectl,
  120. namespace=namespace)
  121. out = run_command(command)
  122. try:
  123. output = json.loads(out)
  124. except Exception as e:
  125. # output = {'namespace': namespace, 'status': e}
  126. output = []
  127. return output
  128. def get_pv(persistentvolume, namespace):
  129. """ return list of persistent volumes in specific namespace """
  130. command = "{command} {namespace} get pv {pv}".format(command=kubectl,
  131. namespace=namespace,
  132. pv=persistentvolume)
  133. out = run_command(command)
  134. try:
  135. output = json.loads(out)
  136. except Exception as e:
  137. output = {'persistentvolume': namespace, 'status': e}
  138. return output
  139. def get_namespaces():
  140. """ return list of namespaces """
  141. command = "{command} {namespace} get ns -o json".format(command=kubectl_exec,
  142. namespace="ceph")
  143. out = run_command(command)
  144. try:
  145. output = json.loads(out)
  146. except Exception as e:
  147. output = {"Error": e}
  148. return output
  149. def get_rbd_list(namespace):
  150. """ return list of rbd in json format """
  151. ceph_mon = get_podname("application=ceph,component=mon", namespace)[0]
  152. command = "{command} {namespace} exec -it {pod} -- rbd ls --format json".format(command=kubectl_exec,
  153. namespace=namespace,
  154. pod=ceph_mon)
  155. out = run_command(command)
  156. logger.debug(out)
  157. try:
  158. output = json.loads(out)
  159. except Exception as e:
  160. output = "{{'namaspace': {}, 'status': {} }}".format(namespace, e)
  161. logger.error("parse json rbd list output: {}".format(e))
  162. return output
  163. def monitoring_pvc():
  164. logger.info("PVCs aren't associated with RBD")
  165. rbds = get_rbd_list("ceph")
  166. logger.info("Gettign list of namespaces")
  167. namespaces = get_namespaces()
  168. for namespace in namespaces['items']:
  169. ns = namespace['metadata']['name']
  170. logger.info("Checking namespace: {}".format(ns))
  171. pvc = get_pvc(ns)
  172. for pv in pvc['items']:
  173. rbd = get_pv(pv['spec']['volumeName'], ns)
  174. rbd = rbd['spec']['rbd']['image']
  175. if rbd not in rbds:
  176. print ("pvc_doesnot_have_rbd:{{namespace={},name={}}} 0".format(ns, pv['metadata']['name']))
  177. def monitoring_rbd():
  178. logger.info("RBD volumes aren't associated with PVC")
  179. r = get_rbd_list("ceph")
  180. rbds = []
  181. namespaces = get_namespaces()
  182. for namespace in namespaces['items']:
  183. ns = namespace['metadata']['name']
  184. pvc = get_pvc(ns)
  185. for pv in pvc['items']:
  186. rbd = get_pv(pv['spec']['volumeName'], ns)
  187. rbd = rbd['spec']['rbd']['image']
  188. rbds.append(rbd)
  189. logger.info(rbd)
  190. for i in r:
  191. if i not in rbds:
  192. logger.debug(i)
  193. print ("rbd_doesnot_have_pvc:{{name={}}} {}".format(i, len(check_rbd_status("ceph", i)['watchers'])))
  194. def monitoring_pv():
  195. logger.info("PVs aren't associated with PVC ")
  196. command = "{config} {kubectl} get pv -o json".format(config=CONFIG, kubectl=KUBECTL)
  197. logger.debug(command)
  198. out = run_command(command)
  199. logger.debug(out)
  200. try:
  201. output = json.loads(out)
  202. for pv in output['items']:
  203. if pv['status']['phase'] == "Released":
  204. print ("pv_released:{{name={},status={}}} 0".format(pv['metadata']['name'], pv['status']['phase']))
  205. except Exception as e:
  206. logger.error("gettign rbd list: {}".format(e))
  207. sys.exit(1)
  208. def monitoring():
  209. monitoring_rbd()
  210. monitoring_pvc()
  211. monitoring_pv()
  212. def main():
  213. # nagios monitoring
  214. # 0 - Service is OK.
  215. # 1 - Service has a WARNING.
  216. # 2 - Service is in a CRITICAL status.
  217. # 3 - Service status is UNKNOWN
  218. # promotheus text file
  219. options = parse_args()
  220. setup_logging(options)
  221. if options.all:
  222. logger.info("all")
  223. monitoring()
  224. if options.rbd:
  225. logger.info("rbd")
  226. monitoring_rbd()
  227. if options.pvc:
  228. logger.info("pvc")
  229. monitoring_pvc()
  230. if options.pv:
  231. logger.info("pv")
  232. monitoring_pv()
  233. sys.exit(1)
  234. if __name__ == "__main__":
  235. main()