Operation documentation
This commit is contained in:
Adam Tengler
BIN
doc/source/operate/figures/graphite_composer.png
Normal file
BIN
doc/source/operate/figures/graphite_composer.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 105 KiB |
BIN
doc/source/operate/figures/graphite_functions.png
Normal file
BIN
doc/source/operate/figures/graphite_functions.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 55 KiB |
BIN
doc/source/operate/figures/heka_message_flow.png
Normal file
BIN
doc/source/operate/figures/heka_message_flow.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 37 KiB |
BIN
doc/source/operate/figures/heka_overview_diagram.png
Normal file
BIN
doc/source/operate/figures/heka_overview_diagram.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
BIN
doc/source/operate/figures/kibana_fields.png
Normal file
BIN
doc/source/operate/figures/kibana_fields.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
BIN
doc/source/operate/figures/kibana_index_pattern.png
Normal file
BIN
doc/source/operate/figures/kibana_index_pattern.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
BIN
doc/source/operate/figures/kibana_save_search.png
Normal file
BIN
doc/source/operate/figures/kibana_save_search.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
296
doc/source/operate/figures/monitoring_system.svg
Normal file
296
doc/source/operate/figures/monitoring_system.svg
Normal file
@@ -0,0 +1,296 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" fill-opacity="1" color-rendering="auto" color-interpolation="auto" text-rendering="auto" stroke="black" stroke-linecap="square" width="3505" stroke-miterlimit="10" shape-rendering="auto" stroke-opacity="1" fill="black" stroke-dasharray="none" font-weight="normal" stroke-width="1" height="2072" font-family="'Dialog'" font-style="normal" stroke-linejoin="miter" font-size="12" stroke-dashoffset="0" image-rendering="auto">
|
||||
<!--Generated by ySVG 2.5-->
|
||||
<defs id="genericDefs"/>
|
||||
<g>
|
||||
<defs id="defs1">
|
||||
<clipPath clipPathUnits="userSpaceOnUse" id="clipPath1">
|
||||
<path d="M0 0 L3505 0 L3505 2072 L0 2072 L0 0 Z"/>
|
||||
</clipPath>
|
||||
<clipPath clipPathUnits="userSpaceOnUse" id="clipPath2">
|
||||
<path d="M-1067 -777 L2438 -777 L2438 1295 L-1067 1295 L-1067 -777 Z"/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
<g fill="white" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="translate(1067,777)" stroke="white">
|
||||
<rect x="-1067" width="3505" height="2072" y="-777" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="-1045.4927" xml:space="preserve" y="986.6396" clip-path="url(#clipPath2)" stroke="none">Storage Device Y</text>
|
||||
<rect x="-1051.4927" y="950.9365" clip-path="url(#clipPath2)" fill="none" width="764.2738" rx="4" ry="4" height="289.8092" stroke-width="5"/>
|
||||
<text x="-1045.4927" xml:space="preserve" y="580.8135" clip-path="url(#clipPath2)" stroke="none">Network Device X</text>
|
||||
<rect x="-1051.4927" y="545.1104" clip-path="url(#clipPath2)" fill="none" width="764.2738" rx="4" ry="4" height="289.8092" stroke-width="5"/>
|
||||
<text x="-1045.4927" xml:space="preserve" y="-10.2523" clip-path="url(#clipPath2)" stroke="none">Server X</text>
|
||||
<rect x="-1051.4927" y="-45.9554" clip-path="url(#clipPath2)" fill="none" width="764.2738" rx="4" ry="4" height="434.5625" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(255,255,153)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(255,255,153)">
|
||||
<rect x="-69.4134" y="150.103" clip-path="url(#clipPath2)" width="649.0154" rx="4" ry="4" height="156.4153" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="127.0162" xml:space="preserve" y="202.1388" clip-path="url(#clipPath2)" stroke="none">service1.role1</text>
|
||||
<text x="236.86" xml:space="preserve" y="239.3888" clip-path="url(#clipPath2)" stroke="none">...</text>
|
||||
<text x="122.5006" xml:space="preserve" y="276.6388" clip-path="url(#clipPath2)" stroke="none">serviceN.roleN</text>
|
||||
<rect x="-69.4134" y="150.103" clip-path="url(#clipPath2)" fill="none" width="649.0154" rx="4" ry="4" height="156.4153" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,255)">
|
||||
<rect x="-69.4134" y="20.5925" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="-24.3236" xml:space="preserve" y="65.6706" clip-path="url(#clipPath2)" stroke="none">sensu-client</text>
|
||||
<rect x="-69.4134" y="20.5925" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,204)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,204)">
|
||||
<rect x="270.0943" y="20.5925" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="296.981" xml:space="preserve" y="65.6706" clip-path="url(#clipPath2)" stroke="none">collectd-client</text>
|
||||
<rect x="270.0943" y="20.5925" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,153,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,153,255)">
|
||||
<rect x="801.8707" y="-627.2733" clip-path="url(#clipPath2)" width="364.4488" rx="4" ry="4" height="111.3746" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="837.1654" xml:space="preserve" y="-560.5079" clip-path="url(#clipPath2)" stroke="none">rabbitmq-server</text>
|
||||
<rect x="801.8707" y="-627.2733" clip-path="url(#clipPath2)" fill="none" width="364.4488" rx="4" ry="4" height="111.3746" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(255,255,153)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(255,255,153)">
|
||||
<rect x="-993.8635" y="146.3982" clip-path="url(#clipPath2)" width="649.0154" rx="4" ry="4" height="163.825" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="-797.434" xml:space="preserve" y="202.1388" clip-path="url(#clipPath2)" stroke="none">service1.role1</text>
|
||||
<text x="-687.5902" xml:space="preserve" y="239.3888" clip-path="url(#clipPath2)" stroke="none">...</text>
|
||||
<text x="-801.9496" xml:space="preserve" y="276.6388" clip-path="url(#clipPath2)" stroke="none">serviceN.roleN</text>
|
||||
<rect x="-993.8635" y="146.3982" clip-path="url(#clipPath2)" fill="none" width="649.0154" rx="4" ry="4" height="163.825" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,255)">
|
||||
<rect x="-993.8635" y="20.5925" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="-948.7737" xml:space="preserve" y="65.6706" clip-path="url(#clipPath2)" stroke="none">sensu-client</text>
|
||||
<rect x="-993.8635" y="20.5925" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,204)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,204)">
|
||||
<rect x="-654.3558" y="20.5925" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="-627.4692" xml:space="preserve" y="65.6706" clip-path="url(#clipPath2)" stroke="none">collectd-client</text>
|
||||
<rect x="-654.3558" y="20.5925" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,255)">
|
||||
<rect x="433.1008" y="934.0398" clip-path="url(#clipPath2)" width="364.4488" rx="4" ry="4" height="111.3746" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="480.1612" xml:space="preserve" y="1000.8052" clip-path="url(#clipPath2)" stroke="none">redis-database</text>
|
||||
<rect x="433.1008" y="934.0398" clip-path="url(#clipPath2)" fill="none" width="364.4488" rx="4" ry="4" height="111.3746" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,255)">
|
||||
<rect x="433.1008" y="699.1663" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="478.1906" xml:space="preserve" y="744.2444" clip-path="url(#clipPath2)" stroke="none">sensu-client</text>
|
||||
<rect x="433.1008" y="699.1663" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,204)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,204)">
|
||||
<rect x="1575.9003" y="-679.8158" clip-path="url(#clipPath2)" width="649.0154" rx="4" ry="4" height="166" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="1733.4783" xml:space="preserve" y="-585.7377" clip-path="url(#clipPath2)" stroke="none">graphite-database</text>
|
||||
<rect x="1575.9003" y="-679.8158" clip-path="url(#clipPath2)" fill="none" width="649.0154" rx="4" ry="4" height="166" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,255)">
|
||||
<rect x="818.5933" y="699.1663" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="856.5894" xml:space="preserve" y="744.2444" clip-path="url(#clipPath2)" stroke="none">sensu-server</text>
|
||||
<rect x="818.5933" y="699.1663" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,204)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,204)">
|
||||
<rect x="1575.9003" y="-483.8158" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="1587.9667" xml:space="preserve" y="-438.7377" clip-path="url(#clipPath2)" stroke="none">carbon-services</text>
|
||||
<rect x="1575.9003" y="-483.8158" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,255)">
|
||||
<rect x="510.2237" y="1132.0784" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="577.9229" xml:space="preserve" y="1177.1565" clip-path="url(#clipPath2)" stroke="none">sensu-api</text>
|
||||
<rect x="510.2237" y="1132.0784" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,204)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,204)">
|
||||
<rect x="1915.408" y="-483.8158" clip-path="url(#clipPath2)" width="309.5077" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="1958.8181" xml:space="preserve" y="-438.7377" clip-path="url(#clipPath2)" stroke="none">graphite-api</text>
|
||||
<rect x="1915.408" y="-483.8158" clip-path="url(#clipPath2)" fill="none" width="309.5077" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(255,255,153)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(255,255,153)">
|
||||
<rect x="-993.8635" y="619.3554" clip-path="url(#clipPath2)" width="649.0154" rx="4" ry="4" height="163.825" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="-797.434" xml:space="preserve" y="675.096" clip-path="url(#clipPath2)" stroke="none">service1.role1</text>
|
||||
<text x="-799.6918" xml:space="preserve" y="712.346" clip-path="url(#clipPath2)" stroke="none">service1.roleN</text>
|
||||
<text x="-801.9496" xml:space="preserve" y="749.596" clip-path="url(#clipPath2)" stroke="none">serviceN.roleN</text>
|
||||
<rect x="-993.8635" y="619.3554" clip-path="url(#clipPath2)" fill="none" width="649.0154" rx="4" ry="4" height="163.825" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(255,204,153)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(255,204,153)">
|
||||
<rect x="1953.6407" y="650.1663" clip-path="url(#clipPath2)" width="468.6239" rx="4" ry="4" height="166" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="2102.7729" xml:space="preserve" y="706.9944" clip-path="url(#clipPath2)" stroke="none">handlers:</text>
|
||||
<text x="1998.5151" xml:space="preserve" y="744.2444" clip-path="url(#clipPath2)" stroke="none">jenkins/salt, hipchat,</text>
|
||||
<text x="2019.3979" xml:space="preserve" y="781.4944" clip-path="url(#clipPath2)" stroke="none">mail, sms, flapjack</text>
|
||||
<rect x="1953.6407" y="650.1663" clip-path="url(#clipPath2)" fill="none" width="468.6239" rx="4" ry="4" height="166" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,255)">
|
||||
<rect x="926.0917" y="1136.1914" clip-path="url(#clipPath2)" width="388.6129" rx="4" ry="4" height="68" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="950.5544" xml:space="preserve" y="1181.2695" clip-path="url(#clipPath2)" stroke="none">horizon-dashboard</text>
|
||||
<rect x="926.0917" y="1136.1914" clip-path="url(#clipPath2)" fill="none" width="388.6129" rx="4" ry="4" height="68" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(204,255,255)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(204,255,255)">
|
||||
<rect x="872.9481" y="934.0398" clip-path="url(#clipPath2)" width="364.4488" rx="4" ry="4" height="111.3746" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="946.6959" xml:space="preserve" y="1000.8052" clip-path="url(#clipPath2)" stroke="none">kedb-server</text>
|
||||
<rect x="872.9481" y="934.0398" clip-path="url(#clipPath2)" fill="none" width="364.4488" rx="4" ry="4" height="111.3746" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(255,255,153)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(255,255,153)">
|
||||
<rect x="-993.8635" y="1013.9286" clip-path="url(#clipPath2)" width="649.0154" rx="4" ry="4" height="163.825" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="-797.434" xml:space="preserve" y="1069.6693" clip-path="url(#clipPath2)" stroke="none">service1.role1</text>
|
||||
<text x="-799.6918" xml:space="preserve" y="1106.9193" clip-path="url(#clipPath2)" stroke="none">service1.roleN</text>
|
||||
<text x="-801.9496" xml:space="preserve" y="1144.1693" clip-path="url(#clipPath2)" stroke="none">serviceN.roleN</text>
|
||||
<rect x="-993.8635" y="1013.9286" clip-path="url(#clipPath2)" fill="none" width="649.0154" rx="4" ry="4" height="163.825" stroke-width="5"/>
|
||||
<text x="278.6052" xml:space="preserve" y="1266.3613" clip-path="url(#clipPath2)" stroke="none">Monitoring service</text>
|
||||
<rect x="272.6052" y="650.1663" clip-path="url(#clipPath2)" fill="none" width="1199.2375" rx="4" ry="4" height="629.7419" stroke-width="5"/>
|
||||
<text x="1522.1124" xml:space="preserve" y="-725.4517" clip-path="url(#clipPath2)" stroke="none">Metrics database</text>
|
||||
<rect x="1516.1124" y="-761.1548" clip-path="url(#clipPath2)" fill="none" width="768.5911" rx="4" ry="4" height="397.8815" stroke-width="5"/>
|
||||
<text x="605.7996" xml:space="preserve" y="-725.4517" clip-path="url(#clipPath2)" stroke="none">Message broker service</text>
|
||||
<text x="660.8621" xml:space="preserve" y="-688.2017" clip-path="url(#clipPath2)" stroke="none">(single/clustered)</text>
|
||||
<rect x="599.7995" y="-761.1548" clip-path="url(#clipPath2)" fill="none" width="768.5911" rx="4" ry="4" height="397.8815" stroke-width="5"/>
|
||||
</g>
|
||||
<g fill="rgb(255,204,153)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" transform="matrix(1,0,0,1,1067,777)" stroke="rgb(255,204,153)">
|
||||
<rect x="1953.6407" y="1087.1914" clip-path="url(#clipPath2)" width="468.6239" rx="4" ry="4" height="166" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" font-weight="bold" stroke-miterlimit="1.45">
|
||||
<text x="1995.9995" xml:space="preserve" y="1144.0195" clip-path="url(#clipPath2)" stroke="none">graphical dashboard:</text>
|
||||
<text x="1976.6401" xml:space="preserve" y="1181.2695" clip-path="url(#clipPath2)" stroke="none">checks, clients, events,</text>
|
||||
<text x="2120.3979" xml:space="preserve" y="1218.5195" clip-path="url(#clipPath2)" stroke="none">metrics</text>
|
||||
<rect x="1953.6407" y="1087.1914" clip-path="url(#clipPath2)" fill="none" width="468.6239" rx="4" ry="4" height="166" stroke-width="5"/>
|
||||
<text x="-121.0426" xml:space="preserve" y="-10.2523" clip-path="url(#clipPath2)" stroke="none">Server X</text>
|
||||
<rect x="-127.0426" y="-45.9554" clip-path="url(#clipPath2)" fill="none" width="764.2738" rx="4" ry="4" height="434.5625" stroke-width="5"/>
|
||||
<text x="-909.0483" font-size="66" y="-598.4545" clip-path="url(#clipPath2)" font-family="sans-serif" stroke="none" font-weight="normal" xml:space="preserve">Monitoring System Infrastructure</text>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke-dasharray="24,8" stroke-width="4" stroke-miterlimit="1.45">
|
||||
<path fill="none" d="M94.3585 88.612 L134.6008 138.3255" clip-path="url(#clipPath2)"/>
|
||||
<path d="M144.1172 150.0816 L140.1481 132.0661 L127.3233 142.4475 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" d="M391.631 88.5854 L342.1003 139.2728" clip-path="url(#clipPath2)"/>
|
||||
<path d="M331.5295 150.0905 L348.9619 144.0552 L337.1608 132.5234 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M134.1346 20.5968 L890.6396 -506.4739" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M904.1779 -515.9062 L884.264 -513.0009 L894.5538 -498.232 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M455.2101 20.5968 L923.3597 -503.5817" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M934.3507 -515.8881 L915.6479 -508.458 L929.0732 -496.4678 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" fill="white" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke="white" stroke-width="5" stroke-miterlimit="1.45">
|
||||
<rect x="473.0953" width="227.5156" height="78.5" y="-166.0515" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" font-family="sans-serif" shape-rendering="geometricPrecision" stroke-miterlimit="1.45">
|
||||
<text x="475.0953" xml:space="preserve" y="-134.3484" clip-path="url(#clipPath2)" stroke="none">sends metrics</text>
|
||||
<text x="498.3063" xml:space="preserve" y="-97.0984" clip-path="url(#clipPath2)" stroke="none">to graphite</text>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke-dasharray="24,8" stroke-width="4" stroke-miterlimit="1.45">
|
||||
<path fill="none" d="M-830.1627 88.5785 L-792.9371 134.6392" clip-path="url(#clipPath2)"/>
|
||||
<path d="M-783.43 146.4027 L-787.3849 128.3841 L-800.2179 138.7555 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M-740.0986 20.5872 L806.3468 -510.5385" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M821.9521 -515.8981 L802.0047 -518.5632 L807.8516 -501.5392 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" font-family="sans-serif" shape-rendering="geometricPrecision" stroke-miterlimit="1.45">
|
||||
<text x="-440.8992" xml:space="preserve" y="-356.1614" clip-path="url(#clipPath2)" stroke="none">performs local service checks</text>
|
||||
<text x="-469.7976" xml:space="preserve" y="-318.9114" clip-path="url(#clipPath2)" stroke="none">sends metrics to carbon-graphite</text>
|
||||
<text x="-438.6258" xml:space="preserve" y="-281.6614" clip-path="url(#clipPath2)" stroke="none">sends events to sensu-server</text>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke-dasharray="24,8" stroke-width="4" stroke-miterlimit="1.45">
|
||||
<path fill="none" d="M-532.8192 88.5854 L-578.7497 135.5886" clip-path="url(#clipPath2)"/>
|
||||
<path d="M-589.3206 146.4063 L-571.8881 140.3711 L-583.6893 128.8392 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M-419.0509 20.5968 L836.9631 -509.4905" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M852.1647 -515.9062 L832.0817 -517.199 L839.0806 -500.6155 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" d="M482.1922 699.1584 L-711.8637 314.8477" clip-path="url(#clipPath2)"/>
|
||||
<path d="M-726.2614 310.2138 L-713.0825 323.1223 L-708.0272 307.4157 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<rect x="-189.734" y="521.0352" clip-path="url(#clipPath2)" fill="white" width="365.1562" height="41.25" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" font-family="sans-serif" shape-rendering="geometricPrecision" stroke-miterlimit="1.45">
|
||||
<text x="-187.734" xml:space="preserve" y="552.7383" clip-path="url(#clipPath2)" stroke="none">remote service checks</text>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke-dasharray="24,8" stroke-width="4" stroke-miterlimit="1.45">
|
||||
<path fill="none" d="M565.4526 699.1785 L314.9613 319.1394" clip-path="url(#clipPath2)"/>
|
||||
<path d="M306.6376 306.5108 L308.8297 324.8277 L322.6064 315.7472 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M598.1694 699.2016 L962.3997 -500.1467" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M967.1943 -515.9348 L953.3522 -501.3268 L970.5754 -496.0962 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M925.9067 767.1625 L706.4203 924.4478" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M693.0085 934.0588 L712.882 930.8896 L702.3972 916.2585 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M994.1952 699.1586 L984.6685 -499.4371" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M984.5374 -515.9366 L975.6807 -497.8656 L993.6802 -498.0087 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" fill="white" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke="white" stroke-width="5" stroke-miterlimit="1.45">
|
||||
<rect x="872.152" width="497.1562" height="78.5" y="-86.4614" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" font-family="sans-serif" shape-rendering="geometricPrecision" stroke-miterlimit="1.45">
|
||||
<text x="935.1364" xml:space="preserve" y="-54.7583" clip-path="url(#clipPath2)" stroke="none">sends checks to clients</text>
|
||||
<text x="874.152" xml:space="preserve" y="-17.5083" clip-path="url(#clipPath2)" stroke="none"> receives results from clients</text>
|
||||
<path fill="none" d="M1166.3149 -541.8644 L1559.6101 -477.7146" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path d="M1575.8949 -475.0584 L1559.5785 -486.8387 L1556.6808 -469.0734 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" d="M1769.9221 -483.8203 L1792.0774 -503.0059" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path d="M1804.5507 -513.8072 L1785.0519 -508.8275 L1796.8351 -495.2204 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" d="M630.9992 1045.3966 L650.9411 1116.2246" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path d="M655.4128 1132.1071 L659.1978 1112.3416 L641.8714 1117.2198 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" d="M1996.2468 -513.8232 L2018.4573 -494.5898" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path d="M2030.9305 -483.7885 L2023.215 -502.3754 L2011.4318 -488.7682 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke-dasharray="24,8" stroke-width="4" stroke-miterlimit="1.45">
|
||||
<path fill="none" d="M630.4448 699.1765 L2015.7434 -406.3862" clip-path="url(#clipPath2)"/>
|
||||
<path d="M2027.5652 -415.8208 L2009.5226 -411.9768 L2019.8148 -399.0803 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<rect x="1116.1098" y="102.4252" clip-path="url(#clipPath2)" fill="white" width="425.7969" height="78.5" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" font-family="sans-serif" shape-rendering="geometricPrecision" stroke-miterlimit="1.45">
|
||||
<text x="1174.2583" xml:space="preserve" y="134.1283" clip-path="url(#clipPath2)" stroke="none">checks graphite for</text>
|
||||
<text x="1118.1099" xml:space="preserve" y="171.3783" clip-path="url(#clipPath2)" stroke="none">metric threshold breaches</text>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke-dasharray="24,8" stroke-width="4" stroke-miterlimit="1.45">
|
||||
<path fill="none" d="M433.1205 729.2403 L-329.7306 709.8849" clip-path="url(#clipPath2)"/>
|
||||
<path d="M-344.8507 709.5013 L-328.5652 718.1672 L-328.1468 701.6725 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M1128.1012 733.1663 L1937.1196 733.1663" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M1953.6196 733.1663 L1935.6196 724.1663 L1935.6196 742.1663 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" font-family="sans-serif" shape-rendering="geometricPrecision" stroke-miterlimit="1.45">
|
||||
<text x="1635.1643" xml:space="preserve" y="637.1302" clip-path="url(#clipPath2)" stroke="none">automated</text>
|
||||
<text x="1602.7502" xml:space="preserve" y="674.3802" clip-path="url(#clipPath2)" stroke="none">event handling</text>
|
||||
<path fill="none" d="M819.7216 1167.476 L909.6262 1168.2878" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path d="M926.1255 1168.4369 L908.2076 1159.2747 L908.045 1177.2739 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" d="M2054.3113 -415.8267 L1321.6863 1155.1649" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path d="M1314.7126 1170.1188 L1330.4769 1157.6093 L1314.1636 1150.0017 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" d="M984.1846 767.1468 L1032.4033 918.335" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path d="M1037.4169 934.0549 L1040.522 914.1713 L1023.373 919.6406 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" fill="white" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke="white" stroke-width="5" stroke-miterlimit="1.45">
|
||||
<rect x="974.6136" width="344.9375" height="78.5" y="811.2144" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" font-family="sans-serif" shape-rendering="geometricPrecision" stroke-miterlimit="1.45">
|
||||
<text x="976.6136" xml:space="preserve" y="842.9175" clip-path="url(#clipPath2)" stroke="none">event mutator - kedb</text>
|
||||
<text x="1004.1761" xml:space="preserve" y="880.1675" clip-path="url(#clipPath2)" stroke="none"> records for event</text>
|
||||
<path fill="none" d="M1075.2848 1045.3732 L1102.5089 1120.6963" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path d="M1108.1176 1136.2137 L1110.4631 1116.2263 L1093.5349 1122.3448 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" shape-rendering="geometricPrecision" stroke-dasharray="24,8" stroke-width="4" stroke-miterlimit="1.45">
|
||||
<path fill="none" d="M470.0296 767.1559 L-370.8463 1009.7284" clip-path="url(#clipPath2)"/>
|
||||
<path d="M-385.3788 1013.9206 L-367.2386 1017.274 L-371.8119 1001.4205 Z" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
<path fill="none" stroke-dasharray="none" d="M1314.7329 1170.1914 L1937.132 1170.1914" stroke-width="5" clip-path="url(#clipPath2)"/>
|
||||
<path stroke-dasharray="none" d="M1953.632 1170.1914 L1935.632 1161.1914 L1935.632 1179.1914 Z" stroke-width="5" clip-path="url(#clipPath2)" stroke="none"/>
|
||||
</g>
|
||||
<g font-size="32" stroke-linecap="butt" transform="matrix(1,0,0,1,1067,777)" text-rendering="geometricPrecision" font-family="sans-serif" shape-rendering="geometricPrecision" stroke-miterlimit="1.45">
|
||||
<text x="1601.2954" xml:space="preserve" y="1070.2705" clip-path="url(#clipPath2)" stroke="none">visual system</text>
|
||||
<text x="1547.2017" xml:space="preserve" y="1107.5205" clip-path="url(#clipPath2)" stroke="none">state representation</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 33 KiB |
@@ -1,7 +1,25 @@
|
||||
|
||||
Event monitoring
|
||||
============================
|
||||
================
|
||||
|
||||
**Monitoring Service (Sensu)**
|
||||
|
||||
Sensu is often described as the “monitoring router”. Essentially, Sensu takes the results of “check” scripts run across many systems, and if certain conditions are met, passes their information to one or more “handlers”. Checks are used, for example, to determine if a service like Apache is up or down. Checks can also be used to collect data, such as MySQL query statistics or Rails application metrics. Handlers take actions, using result information, such as sending an email, messaging a chat room, or adding a data point to a graph. There are several types of handlers, but the most common and most powerful is “pipe”, a script that receives data via standard input. Check and handler scripts can be written in any language, and the community repository continues to grow!
|
||||
|
||||
Sensu properties:
|
||||
|
||||
* Written in Ruby, using EventMachine
|
||||
* Great test coverage with continuous integration via Travis CI
|
||||
* Can use existing Nagios plugins
|
||||
* Configuration all in JSON
|
||||
* Has a message-oriented architecture, using RabbitMQ and JSON payloads
|
||||
* Packages are “omnibus”, for consistency, isolation, and low-friction deployment
|
||||
|
||||
Sensu embraces modern infrastructure design, works elegantly with configuration management tools, and is built for the cloud.
|
||||
|
||||
The Sensu framework contains a number of components. The following diagram depicts these core elements and how they interact with one another.
|
||||
|
||||
.. image:: https://sensuapp.org/docs/latest/img/sensu-diagram-87a902f0.gif
|
||||
|
||||
--------------
|
||||
|
||||
|
||||
@@ -2,6 +2,70 @@
|
||||
Log monitoring
|
||||
============================
|
||||
|
||||
**Log Processing Service (Heka, ElasticSearch)**
|
||||
|
||||
Our logging stack currently contains following services:
|
||||
|
||||
* Heka - log collection, streaming and processing
|
||||
* Rabbitmq - amqp message broker
|
||||
* Elasticsearch - indexed log storage
|
||||
* Kibana - UI for log analysis
|
||||
|
||||
Following diagram show heka message flow thru components
|
||||
|
||||
.. figure :: /operate/figures/heka_message_flow.png
|
||||
:width: 75%
|
||||
:align: center
|
||||
|
||||
Heka
|
||||
****
|
||||
|
||||
Heka is an open source stream processing software system developed by Mozilla.
|
||||
Heka is a “Swiss Army Knife” type tool for data processing, useful for a wide variety of different tasks, such as:
|
||||
|
||||
* Loading and parsing log files from a file system.
|
||||
* Accepting statsd type metrics data for aggregation and forwarding to upstream time series data stores such as graphite or InfluxDB.
|
||||
* Launching external processes to gather operational data from the local system.
|
||||
* Performing real time analysis, graphing, and anomaly detection on any data flowing through the Heka pipeline.
|
||||
* Shipping data from one location to another via the use of an external transport (such as AMQP) or directly (via TCP).
|
||||
* Delivering processed data to one or more persistent data stores.
|
||||
|
||||
Heka overview diagram
|
||||
|
||||
.. figure :: /operate/figures/heka_overview_diagram.png
|
||||
:width: 75%
|
||||
:align: center
|
||||
|
||||
ElasticSearch
|
||||
*************
|
||||
|
||||
Elasticsearch is a search server based on Lucene.It provides a distributed, multitenant-capable full-text search engine with an HTTP web interface and schema-free JSON documents.
|
||||
|
||||
Kibana
|
||||
******
|
||||
|
||||
Kibana is an open source data visualization plugin for Elasticsearch. It provides visualization capabilities on top of the content indexed on an Elasticsearch cluster. Users can create bar, line and scatter plots, or pie charts and maps on top of large volumes of data.
|
||||
|
||||
Kibana dashboard
|
||||
****************
|
||||
|
||||
1. Setup index pattern
|
||||
|
||||
.. figure :: /operate/figures/kibana_index_pattern.png
|
||||
:width: 100%
|
||||
:align: center
|
||||
|
||||
2. Select fields of interrest
|
||||
|
||||
.. figure :: /operate/figures/kibana_fields.png
|
||||
:width: 25%
|
||||
:align: center
|
||||
|
||||
3. Save default search
|
||||
|
||||
.. figure :: /operate/figures/kibana_save_search.png
|
||||
:width: 75%
|
||||
:align: center
|
||||
|
||||
--------------
|
||||
|
||||
|
||||
@@ -2,6 +2,121 @@
|
||||
Meter monitoring
|
||||
============================
|
||||
|
||||
Collectd/Graphite
|
||||
*****************
|
||||
|
||||
Collectd gathers statistics about the system it is running on and stores this information. Those statistics can then be used to find current performance bottlenecks (i.e. performance analysis) and predict future system load (i.e. capacity planning). It's written in C for performance and portability, allowing it to run on systems without scripting language or cron daemon, such as embedded systems. At the same time it includes optimizations and features to handle hundreds of thousands of data sets. It comes with over 90 plugins which range from standard cases to very specialized and advanced topics. It provides powerful networking features and is extensible in numerous ways
|
||||
|
||||
Graphite is an enterprise-scale monitoring tool that runs well on cheap hardware. It was originally designed and written by Chris Davis at Orbitz in 2006 as side project that ultimately grew to be a foundational monitoring tool. In 2008, Orbitz allowed Graphite to be released under the open source Apache 2.0 license. Since then Chris has continued to work on Graphite and has deployed it at other companies including Sears, where it serves as a pillar of the e-commerce monitoring system. Today many large companies use it.
|
||||
|
||||
What Graphite does not do is collect data for you, however there are some tools out there that know how to send data to graphite. Even though it often requires a little code, sending data to Graphite is very simple.
|
||||
|
||||
There are three basic types of meters that are stored in the time-series database.
|
||||
|
||||
* Cumulative: increasing over time (network or disk usage counters)
|
||||
* Gauge: discrete items (number of connected users) and fluctuating values (system load)
|
||||
* Delta: values changing over time (bandwidth)
|
||||
|
||||
Graphite consists of 3 software components:
|
||||
|
||||
* carbon - a Twisted daemon that listens for time-series data
|
||||
* whisper - a simple database library for storing time-series data (similar in design to RRD)
|
||||
* graphite - A Django webapp that renders graphs on-demand using Cairo
|
||||
|
||||
Graphite composer
|
||||
*****************
|
||||
|
||||
Graphite composer is a graphical tool for manipulating graphite metrics and tuning up functions applied to metrics.
|
||||
|
||||
.. figure :: /operate/figures/graphite_composer.png
|
||||
:width: 100%
|
||||
:align: center
|
||||
|
||||
Graphite metrics functions
|
||||
**************************
|
||||
|
||||
The metrics can be adjusted by applying functions on them within the Graphite composer.
|
||||
|
||||
.. figure :: /operate/figures/graphite_functions.png
|
||||
:width: 100%
|
||||
:align: center
|
||||
|
||||
Aside the ability to store time-series data Graphite has a lot of additional functions that can be used to alter time-series data to more appropriate form, if we want to get the delta from the cumulative metrics or ad vice versa.
|
||||
|
||||
**integral(seriesList)**
|
||||
|
||||
This will show the sum over time, sort of like a continuous addition function. Useful for finding totals or trends in metrics that are collected per minute.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
&target=integral(company.sales.perMinute)
|
||||
|
||||
This would start at zero on the left side of the graph, adding the sales each minute, and show the total sales for the time period selected at the right side, (time now, or the time specified by ‘&until=’).
|
||||
|
||||
**derivative(seriesList)**
|
||||
|
||||
This is the opposite of the integral function. This is useful for taking a running total metric and calculating the delta between subsequent data points.
|
||||
|
||||
This function does not normalize for periods of time, as a true derivative would. Instead see the perSecond() function to calculate a rate of change over time.
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
&target=derivative(company.server.application01.ifconfig.TXPackets)
|
||||
|
||||
**sumSeries(*seriesLists)**
|
||||
|
||||
Short form: sum()
|
||||
|
||||
This will add metrics together and return the sum at each datapoint. (See integral for a sum over time)
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
&target=sum(company.server.application*.requestsHandled)
|
||||
|
||||
This would show the sum of all requests handled per minute (provided requestsHandled are collected once a minute). If metrics with different retention rates are combined, the coarsest metric is graphed, and the sum of the other metrics is averaged for the metrics with finer retention rates.
|
||||
|
||||
Read more about functions at http://graphite.readthedocs.org/en/latest/functions.html#module-graphite.render.functions
|
||||
|
||||
Get aggregated CPU usage of a node in percents
|
||||
**********************************************
|
||||
|
||||
In this task we'll look at how to get some useful metric out of data gathered by collectd. The CPU usage is in form of counters. We basically sum value of all states for all CPUs and sum value of idle state for all CPUs and transform these 2 by asPercent function to percentage value. As we are doing ratio there's no need for derivation.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
sumSeries(default_prd.nodename.cpu.*.idle)
|
||||
# sum of all CPU idle state counters
|
||||
|
||||
sumSeries(default_prd.nodename.cpu.*.*)
|
||||
# sum of all CPU state counters
|
||||
|
||||
asPercent(sumSeries(default_prd.nodename.cpu.*.idle), sumSeries(default_prd.nodename.cpu.*.*))
|
||||
# gives percentual ratio of two metrics and in our case the ratio of free cpu, to get ratio of used CPU, the resulting series should be scaled by -1 and ofsetted by 100.
|
||||
|
||||
Send arbitrary metric to Graphite
|
||||
*********************************
|
||||
|
||||
It is possible to send metrics to graphite from a bash script or from within an application.Graphite understands messages with this format:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
metric_path value timestamp\n
|
||||
|
||||
* `metric_path` is the metric namespace that you want to populate.
|
||||
* `value` is the value that you want to assign to the metric at this time.
|
||||
* `timestamp` is the unix epoch time.
|
||||
|
||||
Try send following metric from any node in the cluster:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
root@cfg01:~# echo "test.bash.stats 42 `date +%s`" | nc mon01 2003
|
||||
|
||||
--------------
|
||||
|
||||
|
||||
@@ -5,6 +5,15 @@ Chapter 1. Overview
|
||||
|
||||
.. toctree::
|
||||
|
||||
The overall health of the systems is measured continuously. The metering system collects metrics from the systems and store them in time-series database for furher evaluation and analysys. The log collecting system collects logs from all systems, transforms them to unified form and stores them for analysis. The monitoring system checks for functionality of separate systems and raises events in case of threshold breach. The monitoring systems may query log and time-series databases for accident patterns and raise an event if anomaly is detected.
|
||||
|
||||
.. figure :: /operate/figures/monitoring_system.svg
|
||||
:width: 100%
|
||||
:align: center
|
||||
|
||||
**The difference between monitoring and metering**
|
||||
|
||||
Monitoring is generally used to check for functionality on the overall system and to figure out if the hardware for the overall installation and usage needs to be scaled up. With monitoring, we also do not care that much if we have lost some samples in between. Metering is required for information gathering on usage as a base for resource utilisation. Many monitoring checks are simple meter checks with threshold definitions.
|
||||
|
||||
--------------
|
||||
|
||||
|
||||
@@ -2,6 +2,58 @@
|
||||
Troubleshooting database
|
||||
============================
|
||||
|
||||
MySQL Galera
|
||||
************
|
||||
|
||||
MySQL galera cluster status can be verifed through following command:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
root@ctl01:~# mysql -uroot -pXXXX -e "show status;"
|
||||
...
|
||||
| wsrep_local_state_comment | Synced |
|
||||
| wsrep_cert_index_size | 41 |
|
||||
| wsrep_causal_reads | 0 |
|
||||
| wsrep_incoming_addresses | 10.0.106.72:3306,10.0.106.73:3306,10.0.106.71:3306 |
|
||||
| wsrep_cluster_conf_id | 29 |
|
||||
| wsrep_cluster_size | 3 |
|
||||
...
|
||||
|
||||
Rejoining one node
|
||||
------------------
|
||||
|
||||
MySQL Galera is build from 3 nodes. Failure one of node does not cause any outage of database and should be solved by restarting mysql service. If node cannot be rejoined back to cluster, there must be removed several files:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
rm -rf /var/lib/mysql/grastate*
|
||||
rm -rf /var/lib/mysql/ib_log*
|
||||
service mysql start
|
||||
|
||||
|
||||
Restarting whole cluster
|
||||
------------------------
|
||||
|
||||
In case of outage all three mysql cluster nodes, it must be started with specific order and parameters. At first check that all mysql proceses at all nodes are killed.
|
||||
|
||||
**Node 1** - configure wsrep_cluster_address without any ip addresses and start mysql
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
vim /etc/mysql/my.cnf
|
||||
....
|
||||
wsrep_cluster_address=gcomm://
|
||||
....
|
||||
|
||||
service mysql start
|
||||
|
||||
**Node 2** and **Node 3**
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
rm -rf /var/lib/mysql/grastate*
|
||||
rm -rf /var/lib/mysql/ib_log*
|
||||
service mysql start
|
||||
|
||||
--------------
|
||||
|
||||
|
||||
@@ -2,6 +2,133 @@
|
||||
Troubleshooting networking
|
||||
============================
|
||||
|
||||
OpenContrail
|
||||
************
|
||||
|
||||
Contrail-status provides information status of all contrail services. All of them should be active except contrail-device-manager, contrail-schema and contrail-svc-monitor. These can be in active state at only one node in cluster. It is dynamically switched in case of failure.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
root@ctl01:~# contrail-status
|
||||
|
||||
== Contrail Control ==
|
||||
supervisor-control: active
|
||||
contrail-control active
|
||||
contrail-control-nodemgr active
|
||||
contrail-dns active
|
||||
contrail-named active
|
||||
|
||||
== Contrail Analytics ==
|
||||
supervisor-analytics: active
|
||||
contrail-analytics-api active
|
||||
contrail-analytics-nodemgr active
|
||||
contrail-collector active
|
||||
contrail-query-engine active
|
||||
contrail-snmp-collector active
|
||||
contrail-topology active
|
||||
|
||||
== Contrail Config ==
|
||||
supervisor-config: active
|
||||
contrail-api:0 active
|
||||
contrail-config-nodemgr active
|
||||
contrail-device-manager initializing
|
||||
contrail-discovery:0 active
|
||||
contrail-schema initializing
|
||||
contrail-svc-monitor initializing
|
||||
ifmap active
|
||||
|
||||
== Contrail Web UI ==
|
||||
supervisor-webui: active
|
||||
contrail-webui active
|
||||
contrail-webui-middleware active
|
||||
|
||||
== Contrail Database ==
|
||||
supervisor-database: active
|
||||
contrail-database active
|
||||
contrail-database-nodemgr active
|
||||
|
||||
== Contrail Support Services ==
|
||||
supervisor-support-service: active
|
||||
rabbitmq-server active
|
||||
|
||||
OpenContrail uses for all services python daemon supervisord, which create logical groups from specific services. It is automaticaly installed with contrail packages.
|
||||
|
||||
* supervisor-support-service
|
||||
* supervisor-openstack
|
||||
* supervisor-database
|
||||
* supervisor-config
|
||||
* supervisor-analytics
|
||||
* supervisor-control
|
||||
* supervisor-webui
|
||||
|
||||
Services can be restarted as whole supervisor
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
service supervisor-openstack restart
|
||||
|
||||
or as individual services inside of supervisor
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
root@ctl01:~# supervisorctl -s unix:///tmp/supervisord_support_service.sock status
|
||||
rabbitmq-server RUNNING pid 1335, uptime 2 days, 21:11:55
|
||||
|
||||
root@ctl01:~# supervisorctl -s unix:///tmp/supervisord_openstack.sock status
|
||||
cinder-api RUNNING pid 57685, uptime 2 days, 0:10:39
|
||||
cinder-scheduler RUNNING pid 57675, uptime 2 days, 0:10:44
|
||||
glance-api RUNNING pid 9317, uptime 2 days, 21:08:52
|
||||
glance-registry RUNNING pid 9352, uptime 2 days, 21:08:51
|
||||
heat-api RUNNING pid 9393, uptime 2 days, 21:08:50
|
||||
heat-engine RUNNING pid 9351, uptime 2 days, 21:08:51
|
||||
keystone RUNNING pid 9325, uptime 2 days, 21:08:52
|
||||
nova-api RUNNING pid 9339, uptime 2 days, 21:08:51
|
||||
nova-conductor RUNNING pid 9300, uptime 2 days, 21:08:53
|
||||
nova-console RUNNING pid 9330, uptime 2 days, 21:08:52
|
||||
nova-consoleauth RUNNING pid 9319, uptime 2 days, 21:08:52
|
||||
nova-novncproxy RUNNING pid 9299, uptime 2 days, 21:08:53
|
||||
nova-objectstore RUNNING pid 9321, uptime 2 days, 21:08:52
|
||||
nova-scheduler RUNNING pid 9344, uptime 2 days, 21:08:51
|
||||
|
||||
root@ctl01:~# supervisorctl -s unix:///tmp/supervisord_database.sock status
|
||||
contrail-database RUNNING pid 1349, uptime 2 days, 21:12:33
|
||||
contrail-database-nodemgr RUNNING pid 1347, uptime 2 days, 21:12:33
|
||||
|
||||
root@ctl01:~# supervisorctl -s unix:///tmp/supervisord_config.sock status
|
||||
contrail-api:0 RUNNING pid 49848, uptime 2 days, 20:11:54
|
||||
contrail-config-nodemgr RUNNING pid 49845, uptime 2 days, 20:11:54
|
||||
contrail-device-manager RUNNING pid 49849, uptime 2 days, 20:11:54
|
||||
contrail-discovery:0 RUNNING pid 49847, uptime 2 days, 20:11:54
|
||||
contrail-schema RUNNING pid 49850, uptime 2 days, 20:11:54
|
||||
contrail-svc-monitor RUNNING pid 49851, uptime 2 days, 20:11:54
|
||||
ifmap RUNNING pid 49846, uptime 2 days, 20:11:54
|
||||
|
||||
root@ctl01:~# supervisorctl -s unix:///tmp/supervisord_config.sock status
|
||||
contrail-api:0 RUNNING pid 49848, uptime 2 days, 20:12:08
|
||||
contrail-config-nodemgr RUNNING pid 49845, uptime 2 days, 20:12:08
|
||||
contrail-device-manager RUNNING pid 49849, uptime 2 days, 20:12:08
|
||||
contrail-discovery:0 RUNNING pid 49847, uptime 2 days, 20:12:08
|
||||
contrail-schema RUNNING pid 49850, uptime 2 days, 20:12:08
|
||||
contrail-svc-monitor RUNNING pid 49851, uptime 2 days, 20:12:08
|
||||
ifmap RUNNING pid 49846, uptime 2 days, 20:12:08
|
||||
|
||||
root@ctl01:~# supervisorctl -s unix:///tmp/supervisord_analytics.sock status
|
||||
contrail-analytics-api RUNNING pid 1346, uptime 2 days, 21:13:17
|
||||
contrail-analytics-nodemgr RUNNING pid 1340, uptime 2 days, 21:13:17
|
||||
contrail-collector RUNNING pid 1344, uptime 2 days, 21:13:17
|
||||
contrail-query-engine RUNNING pid 1345, uptime 2 days, 21:13:17
|
||||
contrail-snmp-collector RUNNING pid 1341, uptime 2 days, 21:13:17
|
||||
contrail-topology RUNNING pid 1343, uptime 2 days, 21:13:17
|
||||
|
||||
root@ctl01:~# supervisorctl -s unix:///tmp/supervisord_control.sock status
|
||||
contrail-control RUNNING pid 1330, uptime 2 days, 21:13:29
|
||||
contrail-control-nodemgr RUNNING pid 1328, uptime 2 days, 21:13:29
|
||||
contrail-dns RUNNING pid 1331, uptime 2 days, 21:13:29
|
||||
contrail-named RUNNING pid 1333, uptime 2 days, 21:13:29
|
||||
|
||||
root@ctl01:~# supervisorctl -s unix:///tmp/supervisord_webui.sock status
|
||||
contrail-webui RUNNING pid 1339, uptime 2 days, 21:13:44
|
||||
contrail-webui-middleware RUNNING pid 1342, uptime 2 days, 21:13:44
|
||||
|
||||
--------------
|
||||
|
||||
|
||||
Reference in New Issue
Block a user