import logging import os import time import unittest from tests.common import get_check class VarnishTestCase(unittest.TestCase): def setUp(self): self.v_dump = """client_conn 42561 1.84 Client connections accepted client_drop 0 0.00 Connection dropped, no sess/wrk client_req 1564940 67.82 Client requests received cache_hit 52397 2.27 Cache hits cache_hitpass 0 0.00 Cache hits for pass cache_miss 5395 0.23 Cache misses backend_conn 20191 0.88 Backend conn. success backend_unhealthy 0 0.00 Backend conn. not attempted backend_busy 0 0.00 Backend conn. too many backend_fail 1717 0.07 Backend conn. failures backend_reuse 1490649 64.60 Backend conn. reuses backend_toolate 771 0.03 Backend conn. was closed backend_recycle 1491423 64.64 Backend conn. recycles backend_retry 13 0.00 Backend conn. retry fetch_head 5 0.00 Fetch head fetch_length 1052795 45.63 Fetch with Length fetch_chunked 457998 19.85 Fetch chunked fetch_eof 0 0.00 Fetch EOF fetch_bad 0 0.00 Fetch had bad headers fetch_close 13 0.00 Fetch wanted close fetch_oldhttp 0 0.00 Fetch pre HTTP/1.1 closed fetch_zero 0 0.00 Fetch zero len fetch_failed 0 0.00 Fetch failed fetch_1xx 0 0.00 Fetch no body (1xx) fetch_204 0 0.00 Fetch no body (204) fetch_304 0 0.00 Fetch no body (304) n_sess_mem 547 . N struct sess_mem n_sess 29 . N struct sess n_object 149 . N struct object n_vampireobject 0 . N unresurrected objects n_objectcore 168 . N struct objectcore n_objecthead 211 . N struct objecthead n_waitinglist 980 . N struct waitinglist n_vbc 3 . N struct vbc n_wrk 21 . N worker threads n_wrk_create 1999 0.09 N worker threads created n_wrk_failed 0 0.00 N worker threads not created n_wrk_max 313745 13.60 N worker threads limited n_wrk_lqueue 0 0.00 work request queue length n_wrk_queued 7125 0.31 N queued work requests n_wrk_drop 0 0.00 N dropped work requests n_backend 1 . N backends n_expired 5237 . N expired objects n_lru_nuked 0 . N LRU nuked objects n_lru_moved 34482 . N LRU moved objects losthdr 0 0.00 HTTP header overflows n_objsendfile 0 0.00 Objects sent with sendfile n_objwrite 1611740 69.85 Objects sent with write n_objoverflow 0 0.00 Objects overflowing workspace s_sess 42561 1.84 Total Sessions s_req 1564940 67.82 Total Requests s_pipe 0 0.00 Total pipe s_pass 1507148 65.32 Total pass s_fetch 1510811 65.48 Total fetch s_hdrbytes 462276984 20034.54 Total header bytes s_bodybytes 26560298859 1151092.09 Total body bytes sess_closed 2198 0.10 Session Closed sess_pipeline 0 0.00 Session Pipeline sess_readahead 0 0.00 Session Read Ahead sess_linger 1563076 67.74 Session Linger sess_herd 1285271 55.70 Session herd shm_records 114040454 4942.38 SHM records shm_writes 6024350 261.09 SHM writes shm_flushes 0 0.00 SHM flushes due to overflow shm_cont 1353 0.06 SHM MTX contention shm_cycles 46 0.00 SHM cycles through buffer sms_nreq 1732 0.08 SMS allocator requests sms_nobj 0 . SMS outstanding allocations sms_nbytes 0 . SMS outstanding bytes sms_balloc 723976 . SMS bytes allocated sms_bfree 723976 . SMS bytes freed backend_req 1510831 65.48 Backend requests made n_vcl 1 0.00 N vcl total n_vcl_avail 1 0.00 N vcl available n_vcl_discard 0 0.00 N vcl discarded n_ban 1 . N total active bans n_ban_add 1 0.00 N new bans added n_ban_retire 0 0.00 N old bans deleted n_ban_obj_test 0 0.00 N objects tested n_ban_re_test 0 0.00 N regexps tested against n_ban_dups 0 0.00 N duplicate bans removed hcb_nolock 57795 2.50 HCB Lookups without lock hcb_lock 3909 0.17 HCB Lookups with lock hcb_insert 3909 0.17 HCB Inserts esi_errors 0 0.00 ESI parse errors (unlock) esi_warnings 0 0.00 ESI parse warnings (unlock) accept_fail 0 0.00 Accept failures client_drop_late 0 0.00 Connection dropped late uptime 23074 1.00 Client uptime dir_dns_lookups 0 0.00 DNS director lookups dir_dns_failed 0 0.00 DNS director failed lookups dir_dns_hit 0 0.00 DNS director cached lookups hit dir_dns_cache_full 0 0.00 DNS director full dnscache vmods 0 . Loaded VMODs n_gzip 0 0.00 Gzip operations n_gunzip 0 0.00 Gunzip operations LCK.sms.creat 1 0.00 Created locks LCK.sms.destroy 0 0.00 Destroyed locks LCK.sms.locks 5196 0.23 Lock Operations LCK.sms.colls 0 0.00 Collisions LCK.smp.creat 0 0.00 Created locks LCK.smp.destroy 0 0.00 Destroyed locks LCK.smp.locks 0 0.00 Lock Operations LCK.smp.colls 0 0.00 Collisions LCK.sma.creat 1 0.00 Created locks LCK.sma.destroy 0 0.00 Destroyed locks LCK.sma.locks 6474775 280.61 Lock Operations LCK.sma.colls 0 0.00 Collisions LCK.smf.creat 1 0.00 Created locks LCK.smf.destroy 0 0.00 Destroyed locks LCK.smf.locks 30590 1.33 Lock Operations LCK.smf.colls 0 0.00 Collisions LCK.hsl.creat 0 0.00 Created locks LCK.hsl.destroy 0 0.00 Destroyed locks LCK.hsl.locks 0 0.00 Lock Operations LCK.hsl.colls 0 0.00 Collisions LCK.hcb.creat 1 0.00 Created locks LCK.hcb.destroy 0 0.00 Destroyed locks LCK.hcb.locks 7801 0.34 Lock Operations LCK.hcb.colls 0 0.00 Collisions LCK.hcl.creat 0 0.00 Created locks LCK.hcl.destroy 0 0.00 Destroyed locks LCK.hcl.locks 0 0.00 Lock Operations LCK.hcl.colls 0 0.00 Collisions LCK.vcl.creat 1 0.00 Created locks LCK.vcl.destroy 0 0.00 Destroyed locks LCK.vcl.locks 3987 0.17 Lock Operations LCK.vcl.colls 0 0.00 Collisions LCK.stat.creat 1 0.00 Created locks LCK.stat.destroy 0 0.00 Destroyed locks LCK.stat.locks 547 0.02 Lock Operations LCK.stat.colls 0 0.00 Collisions LCK.sessmem.creat 1 0.00 Created locks LCK.sessmem.destroy 0 0.00 Destroyed locks LCK.sessmem.locks 43217 1.87 Lock Operations LCK.sessmem.colls 0 0.00 Collisions LCK.wstat.creat 1 0.00 Created locks LCK.wstat.destroy 0 0.00 Destroyed locks LCK.wstat.locks 74248 3.22 Lock Operations LCK.wstat.colls 0 0.00 Collisions LCK.herder.creat 1 0.00 Created locks LCK.herder.destroy 0 0.00 Destroyed locks LCK.herder.locks 5250 0.23 Lock Operations LCK.herder.colls 0 0.00 Collisions LCK.wq.creat 2 0.00 Created locks LCK.wq.destroy 0 0.00 Destroyed locks LCK.wq.locks 2618934 113.50 Lock Operations LCK.wq.colls 0 0.00 Collisions LCK.objhdr.creat 4840 0.21 Created locks LCK.objhdr.destroy 4629 0.20 Destroyed locks LCK.objhdr.locks 243135 10.54 Lock Operations LCK.objhdr.colls 0 0.00 Collisions LCK.exp.creat 1 0.00 Created locks LCK.exp.destroy 0 0.00 Destroyed locks LCK.exp.locks 33597 1.46 Lock Operations LCK.exp.colls 0 0.00 Collisions LCK.lru.creat 2 0.00 Created locks LCK.lru.destroy 0 0.00 Destroyed locks LCK.lru.locks 5386 0.23 Lock Operations LCK.lru.colls 0 0.00 Collisions LCK.cli.creat 1 0.00 Created locks LCK.cli.destroy 0 0.00 Destroyed locks LCK.cli.locks 7696 0.33 Lock Operations LCK.cli.colls 0 0.00 Collisions LCK.ban.creat 1 0.00 Created locks LCK.ban.destroy 0 0.00 Destroyed locks LCK.ban.locks 33611 1.46 Lock Operations LCK.ban.colls 0 0.00 Collisions LCK.vbp.creat 1 0.00 Created locks LCK.vbp.destroy 0 0.00 Destroyed locks LCK.vbp.locks 0 0.00 Lock Operations LCK.vbp.colls 0 0.00 Collisions LCK.vbe.creat 1 0.00 Created locks LCK.vbe.destroy 0 0.00 Destroyed locks LCK.vbe.locks 43813 1.90 Lock Operations LCK.vbe.colls 0 0.00 Collisions LCK.backend.creat 1 0.00 Created locks LCK.backend.destroy 0 0.00 Destroyed locks LCK.backend.locks 3050125 132.19 Lock Operations LCK.backend.colls 0 0.00 Collisions SMF.s0.c_req 12925 0.56 Allocator requests SMF.s0.c_fail 0 0.00 Allocator failures SMF.s0.c_bytes 812273664 35202.98 Bytes allocated SMF.s0.c_freed 799502336 34649.49 Bytes freed SMF.s0.g_alloc 311 . Allocations outstanding SMF.s0.g_bytes 12771328 . Bytes outstanding SMF.s0.g_space 524099584 . Bytes available SMF.s0.g_smf 383 . N struct smf SMF.s0.g_smf_frag 60 . N small free smf SMF.s0.g_smf_large 12 . N large free smf SMA.Transient.c_req 3010912 130.49 Allocator requests SMA.Transient.c_fail 0 0.00 Allocator failures SMA.Transient.c_bytes 63026331110 2731487.00 Bytes allocated SMA.Transient.c_freed 63026331110 2731487.00 Bytes freed SMA.Transient.g_alloc 0 . Allocations outstanding SMA.Transient.g_bytes 0 . Bytes outstanding SMA.Transient.g_space 0 . Bytes available VBE.default(127.0.0.1,,8080).vcls 1 . VCL references VBE.default(127.0.0.1,,8080).happy 0 . Happy health probes""" self.xml_dump = """ client_conn 475607 a Client connections accepted client_drop 0 a Connection dropped, no sess/wrk client_req 46007888 a Client requests received cache_hit 29652727 a Cache hits cache_hitpass 0 a Cache hits for pass cache_miss 11551328 a Cache misses backend_conn 187918 a Backend conn. success backend_unhealthy 0 a Backend conn. not attempted backend_busy 0 a Backend conn. too many backend_fail 912 a Backend conn. failures backend_reuse 16078929 a Backend conn. reuses backend_toolate 13560 a Backend conn. was closed backend_recycle 16092503 a Backend conn. recycles backend_retry 45 a Backend conn. retry fetch_head 0 a Fetch head fetch_length 12638932 a Fetch with Length fetch_chunked 3627795 a Fetch chunked fetch_eof 0 a Fetch EOF fetch_bad 0 a Fetch had bad headers fetch_close 36 a Fetch wanted close fetch_oldhttp 0 a Fetch pre HTTP/1.1 closed fetch_zero 0 a Fetch zero len fetch_failed 9 a Fetch failed fetch_1xx 0 a Fetch no body (1xx) fetch_204 0 a Fetch no body (204) fetch_304 0 a Fetch no body (304) n_sess_mem 334 i N struct sess_mem n_sess 31 i N struct sess n_object 391072 i N struct object n_vampireobject 0 i N unresurrected objects n_objectcore 391086 i N struct objectcore n_objecthead 119540 i N struct objecthead n_waitinglist 42255 i N struct waitinglist n_vbc 13 i N struct vbc n_wrk 24 i N worker threads n_wrk_create 3718 a N worker threads created n_wrk_failed 0 a N worker threads not created n_wrk_max 0 a N worker threads limited n_wrk_lqueue 0 a work request queue length n_wrk_queued 37387 a N queued work requests n_wrk_drop 0 a N dropped work requests n_backend 17 i N backends n_expired 5509942 i N expired objects n_lru_nuked 5650303 i N LRU nuked objects n_lru_moved 14960592 i N LRU moved objects losthdr 0 a HTTP header overflows n_objsendfile 0 a Objects sent with sendfile n_objwrite 46952631 a Objects sent with write n_objoverflow 0 a Objects overflowing workspace s_sess 475606 a Total Sessions s_req 46007888 a Total Requests s_pipe 0 a Total pipe s_pass 4715470 a Total pass s_fetch 16266754 a Total fetch s_hdrbytes 13489812656 a Total header bytes s_bodybytes 870366768630 a Total body bytes sess_closed 90657 a Session Closed sess_pipeline 0 a Session Pipeline sess_readahead 0 a Session Read Ahead sess_linger 45917694 a Session Linger sess_herd 15876611 a Session herd shm_records 2312772697 a SHM records shm_writes 97762309 a SHM writes shm_flushes 3 a SHM flushes due to overflow shm_cont 31213 a SHM MTX contention shm_cycles 967 a SHM cycles through buffer sms_nreq 88407 a SMS allocator requests sms_nobj 0 i SMS outstanding allocations sms_nbytes 0 i SMS outstanding bytes sms_balloc 477828 i SMS bytes allocated sms_bfree 477828 i SMS bytes freed backend_req 16266804 a Backend requests made n_vcl 1 a N vcl total n_vcl_avail 1 a N vcl available n_vcl_discard 0 a N vcl discarded n_ban 1 i N total active bans n_ban_add 1 a N new bans added n_ban_retire 0 a N old bans deleted n_ban_obj_test 0 a N objects tested n_ban_re_test 0 a N regexps tested against n_ban_dups 0 a N duplicate bans removed hcb_nolock 41204059 a HCB Lookups without lock hcb_lock 3053198 a HCB Lookups with lock hcb_insert 3053197 a HCB Inserts esi_errors 0 a ESI parse errors (unlock) esi_warnings 0 a ESI parse warnings (unlock) accept_fail 0 a Accept failures client_drop_late 0 a Connection dropped late uptime 265069 a Client uptime dir_dns_lookups 0 a DNS director lookups dir_dns_failed 0 a DNS director failed lookups dir_dns_hit 0 a DNS director cached lookups hit dir_dns_cache_full 0 a DNS director full dnscache vmods 0 i Loaded VMODs n_gzip 0 a Gzip operations n_gunzip 0 a Gunzip operations LCK sms creat 1 a Created locks LCK sms destroy 0 a Destroyed locks LCK sms locks 265221 a Lock Operations LCK sms colls 0 a Collisions LCK smp creat 0 a Created locks LCK smp destroy 0 a Destroyed locks LCK smp locks 0 a Lock Operations LCK smp colls 0 a Collisions LCK sma creat 2 a Created locks LCK sma destroy 0 a Destroyed locks LCK sma locks 74266910 a Lock Operations LCK sma colls 0 a Collisions LCK smf creat 0 a Created locks LCK smf destroy 0 a Destroyed locks LCK smf locks 0 a Lock Operations LCK smf colls 0 a Collisions LCK hsl creat 0 a Created locks LCK hsl destroy 0 a Destroyed locks LCK hsl locks 0 a Lock Operations LCK hsl colls 0 a Collisions LCK hcb creat 1 a Created locks LCK hcb destroy 0 a Destroyed locks LCK hcb locks 5991389 a Lock Operations LCK hcb colls 0 a Collisions LCK hcl creat 0 a Created locks LCK hcl destroy 0 a Destroyed locks LCK hcl locks 0 a Lock Operations LCK hcl colls 0 a Collisions LCK vcl creat 1 a Created locks LCK vcl destroy 0 a Destroyed locks LCK vcl locks 82036 a Lock Operations LCK vcl colls 0 a Collisions LCK stat creat 1 a Created locks LCK stat destroy 0 a Destroyed locks LCK stat locks 334 a Lock Operations LCK stat colls 0 a Collisions LCK sessmem creat 1 a Created locks LCK sessmem destroy 0 a Destroyed locks LCK sessmem locks 480320 a Lock Operations LCK sessmem colls 0 a Collisions LCK wstat creat 1 a Created locks LCK wstat destroy 0 a Destroyed locks LCK wstat locks 1636233 a Lock Operations LCK wstat colls 0 a Collisions LCK herder creat 1 a Created locks LCK herder destroy 0 a Destroyed locks LCK herder locks 35720 a Lock Operations LCK herder colls 0 a Collisions LCK wq creat 2 a Created locks LCK wq destroy 0 a Destroyed locks LCK wq locks 32549458 a Lock Operations LCK wq colls 0 a Collisions LCK objhdr creat 3056380 a Created locks LCK objhdr destroy 2936840 a Destroyed locks LCK objhdr locks 195679071 a Lock Operations LCK objhdr colls 0 a Collisions LCK exp creat 1 a Created locks LCK exp destroy 0 a Destroyed locks LCK exp locks 22976360 a Lock Operations LCK exp colls 0 a Collisions LCK lru creat 2 a Created locks LCK lru destroy 0 a Destroyed locks LCK lru locks 17201617 a Lock Operations LCK lru colls 0 a Collisions LCK cli creat 1 a Created locks LCK cli destroy 0 a Destroyed locks LCK cli locks 88310 a Lock Operations LCK cli colls 0 a Collisions LCK ban creat 1 a Created locks LCK ban destroy 0 a Destroyed locks LCK ban locks 22976610 a Lock Operations LCK ban colls 0 a Collisions LCK vbp creat 1 a Created locks LCK vbp destroy 0 a Destroyed locks LCK vbp locks 1412365 a Lock Operations LCK vbp colls 0 a Collisions LCK vbe creat 1 a Created locks LCK vbe destroy 0 a Destroyed locks LCK vbe locks 377647 a Lock Operations LCK vbe colls 0 a Collisions LCK backend creat 17 a Created locks LCK backend destroy 0 a Destroyed locks LCK backend locks 52925585 a Lock Operations LCK backend colls 0 a Collisions SMA s0 c_req 26057659 a Allocator requests SMA s0 c_fail 732800073660 a Allocator failures SMA s0 c_bytes 506580500540 a Bytes allocated SMA s0 c_freed 500138170202 a Bytes freed SMA s0 g_alloc 791369 i Allocations outstanding SMA s0 g_bytes 6442330338 i Bytes outstanding SMA s0 g_space 120606 i Bytes available SMA Transient c_req 12490635 a Allocator requests SMA Transient c_fail 0 a Allocator failures SMA Transient c_bytes 44213239981 a Bytes allocated SMA Transient c_freed 44213169757 a Bytes freed SMA Transient g_alloc 192 i Allocations outstanding SMA Transient g_bytes 70224 i Bytes outstanding SMA Transient g_space 0 i Bytes available VBE default(127.0.0.1,,80) vcls 1 i VCL references VBE default(127.0.0.1,,80) happy 0 b Happy health probes VBE media_video_prd_services_01(10.93.67.16,,8080) vcls 1 i VCL references VBE media_video_prd_services_01(10.93.67.16,,8080) happy 18446744073709551615 b Happy health probes VBE media_video_prd_services_02(10.93.67.17,,8080) vcls 1 i VCL references VBE media_video_prd_services_02(10.93.67.17,,8080) happy 18446744073709551615 b Happy health probes VBE media_video_prd_services_03(10.93.67.18,,8080) vcls 1 i VCL references VBE media_video_prd_services_03(10.93.67.18,,8080) happy 18446744073709551615 b Happy health probes VBE media_video_prd_services_04(10.93.67.105,,8080) vcls 1 i VCL references VBE media_video_prd_services_04(10.93.67.105,,8080) happy 18446744073709551615 b Happy health probes VBE media_video_prd_services_06(10.93.67.133,,8080) vcls 1 i VCL references VBE media_video_prd_services_06(10.93.67.133,,8080) happy 18446744073709551615 b Happy health probes VBE media_video_prd_services_07(10.93.67.134,,8080) vcls 1 i VCL references VBE media_video_prd_services_07(10.93.67.134,,8080) happy 18446744073709551615 b Happy health probes VBE media_article_prd_services_01(10.93.67.75,,8080) vcls 1 i VCL references VBE media_article_prd_services_01(10.93.67.75,,8080) happy 18446744073709551615 b Happy health probes VBE media_article_prd_services_02(10.93.67.76,,8080) vcls 1 i VCL references VBE media_article_prd_services_02(10.93.67.76,,8080) happy 18446744073709551615 b Happy health probes VBE media_article_prd_services_03(10.93.67.77,,8080) vcls 1 i VCL references VBE media_article_prd_services_03(10.93.67.77,,8080) happy 18446744073709551615 b Happy health probes VBE media_article_prd_services_04(10.93.67.121,,8080) vcls 1 i VCL references VBE media_article_prd_services_04(10.93.67.121,,8080) happy 18446744073709551615 b Happy health probes VBE media_article_prd_services_05(10.93.67.122,,8080) vcls 1 i VCL references VBE media_article_prd_services_05(10.93.67.122,,8080) happy 18446744073709551615 b Happy health probes VBE media_object_prd_services_01(10.93.66.132,,8080) vcls 1 i VCL references VBE media_object_prd_services_01(10.93.66.132,,8080) happy 18446744073709551615 b Happy health probes VBE media_object_prd_services_02(10.93.66.150,,8080) vcls 1 i VCL references VBE media_object_prd_services_02(10.93.66.150,,8080) happy 18446744073709551615 b Happy health probes VBE media_object_prd_services_03(10.93.67.107,,8080) vcls 1 i VCL references VBE media_object_prd_services_03(10.93.67.107,,8080) happy 18446744073709551615 b Happy health probes VBE media_object_prd_services_04(10.93.67.125,,8080) vcls 1 i VCL references VBE media_object_prd_services_04(10.93.67.125,,8080) happy 18446744073709551615 b Happy health probes VBE media_object_prd_services_05(10.93.67.126,,8080) vcls 1 i VCL references VBE media_object_prd_services_05(10.93.67.126,,8080) happy 18446744073709551615 b Happy health probes """ self.config = """ init_config: instances: - varnishstat: /usr/bin/varnishstat """ def testParsing(self): v, instances = get_check('varnish', self.config) v._parse_varnishstat(self.v_dump, False) metrics = v.get_metrics() self.assertEqual([m[2] for m in metrics if m[0] == "varnish.n_waitinglist"][0], 980) assert "varnish.fetch_length" not in [m[0] for m in metrics] # XML parsing v._parse_varnishstat(self.xml_dump, True) metrics = v.get_metrics() self.assertEqual([m[2] for m in metrics if m[0] == "varnish.SMA.s0.g_space"][0], 120606) assert "varnish.SMA.transient.c_bytes" not in [m[0] for m in metrics] def testCheck(self): v, instances = get_check('varnish', self.config) import pprint try: for i in range(3): v.check({"varnishstat": os.popen("which varnishstat").read()[:-1]}) pprint.pprint(v.get_metrics()) time.sleep(1) except Exception: pass def testOldConfig(self): v, instances = get_check('varnish', self.config) config = { 'varnishstat': '/usr/bin/varnishstat' } instances = v.parse_agent_config(config)['instances'] assert instances[0]['varnishstat'] == config['varnishstat'] if __name__ == '__main__': unittest.main()