x/broadview-collector
Code Issues Proposed changes

Add packet trace handler support (handler, serializer for Monasca, plugin integration, and simulators)

Browse Source 
Change-Id: I39caaaf8346ec1aeca7fbeff7611151a1f6d57af
This commit is contained in:
Vitaliy Kharechko 2016-07-12 10:52:34 +03:00
parent 45a05ad71d
commit 85229cc7fa
16 changed files with 1734 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
broadview_collector/handlers/broadviewhandlerbase.py
View File

@ -20,7 +20,7 @@ class BroadViewHandlerBase(object):
'''
return a 2-tuple (data, handled)
where data is understandable by a downstream serializer
where data is understandable by a downstream parser
(in the case of BST, this would be a bst_parser object),
and handled is a boolean, True if the POST was recognized,
and False if not.

1
broadview_collector/handlers/bsthandler.py
View File

@ -25,7 +25,6 @@ class BroadViewHandler(BroadViewHandlerBase):
handled = parser.process(data)
except:
handled = False
print handled
return (parser, handled)
def __repr__(self):

32
broadview_collector/handlers/pthandler.py Normal file
View File

@ -0,0 +1,32 @@
# (C) Copyright Broadcom Corporation 2016
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from broadviewhandlerbase import BroadViewHandlerBase
from broadview_lib.pt.pt_parser import PTParser
class BroadViewHandler(BroadViewHandlerBase):
def __init__(self):
pass
def handlePOST(self, path, ctype, length, data):
parser = PTParser()
try:
handled = parser.process(data)
except:
handled = False
return (parser, handled)
def __repr__(self):
return "PT Handler"

9
broadview_collector/plugins/broadviewpublisherbase.py
View File

@ -12,6 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from broadview_lib.pt.pt_parser import PTParser
from broadview_lib.bst.bst_parser import BSTParser
class BroadViewPublisherBase(object):
def __init__(self):
pass
@ -19,3 +22,9 @@ class BroadViewPublisherBase(object):
def publish(self, host, data):
raise NotImplementedError
def isBST(self, parser):
return isinstance(parser, BSTParser)
def isPT(self, parser):
return isinstance(parser, PTParser)

24
broadview_collector/plugins/kafkapublisher.py
View File

@ -39,16 +39,16 @@ class BroadViewPublisher(BroadViewPublisherBase):
def getKafkaProducer(self):
try:
self._producer = kafka.KafkaProducer(bootstrap_servers=['{}:{}'.format(self._ip_address, self._port)])
self._producer = kafka.KafkaProducer(bootstrap_servers=['{}:{}'.format(self._ip_address, self._port)])
except kafka.errors.NoBrokersAvailable as e:
LOG.error("BroadViewPublisher: NoBrokersAvailable {}".format(e))
except:
LOG.error("Unexpected error: {}".format(sys.exc_info()[0]))
def __init__(self):
self._ip_address = "127.0.0.1"
self._port = "9092"
self._topic = "broadview-bst"
self._ip_address = "127.0.0.1"
self._port = "9092"
self._topic = "broadview-bst"
self.readConfig()
self._producer = None
@ -58,13 +58,21 @@ class BroadViewPublisher(BroadViewPublisherBase):
if not self._producer:
self.getKafkaProducer()
if self._producer:
code = 200
code = 200
if self.isBST(data):
self._topic = "broadview-bst"
success, sdata = BSTToMonasca().serialize(host, data)
sdata = json.loads(sdata)
elif self.isPT(data):
self._topic = "broadview-pt"
success, sdata = PTToMonasca().serialize(host, data)
else:
success = False
if success:
sdata = json.loads(sdata)
if success:
for x in sdata:
for x in sdata:
try:
self._producer.send(self._topic, json.dumps(x))
self._producer.send(self._topic, json.dumps(x))
except:
LOG.info('unable to send to kafka topic {}: {}'.format(self._topic, sys.exc_info()[0]))
else:

14
broadview_collector/plugins/logpublisher.py
View File

@ -18,6 +18,7 @@ from broadviewpublisherbase import BroadViewPublisherBase
# so use it.
from broadview_collector.serializers.bst_to_monasca import BSTToMonasca
from broadview_collector.serializers.pt_to_monasca import PTToMonasca
import json
try:
from oslo_log import log
@ -40,7 +41,7 @@ class BroadViewPublisher(BroadViewPublisherBase):
try:
self._f = open(self._logfile, "w+")
except:
LOG.info("log publisher: unable to open log file {}".format(self.__logfile))
LOG.info("log publisher: unable to open log file {}".format(self._logfile))
def __init__(self):
LOG.info("log publisher: init")
@ -53,11 +54,16 @@ class BroadViewPublisher(BroadViewPublisherBase):
self._f.close()
def publish(self, host, data):
LOG.info("log publisher: publish")
code = 200
success, sdata = BSTToMonasca().serialize(host, data)
sdata = json.loads(sdata)
if self.isBST(data):
success, sdata = BSTToMonasca().serialize(host, data)
elif self.isPT(data):
success, sdata = PTToMonasca().serialize(host, data)
else:
LOG.info("log publisher is not PT or BST")
success = False
if success:
sdata = json.loads(sdata)
for x in sdata:
print >>self._f, json.dumps(x)
self._f.flush()

20
broadview_collector/plugins/monascapublisher.py
View File

@ -16,6 +16,7 @@ from broadviewpublisherbase import BroadViewPublisherBase
from monascaclient import client
import monascaclient.exc as exc
from broadview_collector.serializers.bst_to_monasca import BSTToMonasca
from broadview_collector.serializers.pt_to_monasca import PTToMonasca
import json
import ConfigParser
@ -45,7 +46,6 @@ class BroadViewPublisher(BroadViewPublisherBase):
def __init__(self):
self.readConfig()
try:
self._auth_kwargs = {
'username': self._username,
@ -65,15 +65,21 @@ class BroadViewPublisher(BroadViewPublisherBase):
def publish(self, host, data):
code = 500
if self._monasca_client:
code = 200
success, sdata = BSTToMonasca().serialize(host, data)
sdata = json.loads(sdata)
if self.isBST(data):
success, sdata = BSTToMonasca().serialize(host, data)
elif self.isPT(data):
success, sdata = PTToMonasca().serialize(host, data)
else:
success = False
sdata = None
if success:
for x in sdata:
code = 200
sdata = json.loads(sdata)
for x in sdata:
try:
resp = self._monasca_client.metrics.create(**x)
if not resp.status_code == 200 and not resp.status_code == 204:
code = resp.status_code
code = resp.status_code
break
except exc.HTTPException as he:
LOG.error('HTTPException code=%s message=%s' % (he.code, he.message))
@ -82,5 +88,5 @@ class BroadViewPublisher(BroadViewPublisherBase):
return code
def __repr__(self):
return "BroadView Monasca Publisher {_endpoint} {_api_version}".format(**__dict__(self))
return "BroadView Monasca Publisher {}".format(self.__dict__)

6
broadview_collector/plugins/stacklight.py
View File

@ -54,10 +54,12 @@ class BroadViewPublisher(BroadViewPublisherBase):
def publish(self, host, data):
code = 500
success, sdata = BSTToStacklight().serialize(host, data)
success = False
if self.isBST(data):
success, sdata = BSTToStacklight().serialize(host, data)
if success:
sdata = json.loads(sdata)
for x in sdata:
for x in sdata:
try:
r = requests.post('http://{}:{}'.format(self._ipaddr, self._port), json=sdata, timeout=self._timeout)
code = r.status_code

11
broadview_collector/plugins/syslogpublisher.py
View File

@ -18,6 +18,7 @@ from broadviewpublisherbase import BroadViewPublisherBase
# so use it.
from broadview_collector.serializers.bst_to_monasca import BSTToMonasca
from broadview_collector.serializers.pt_to_monasca import PTToMonasca
import json
import syslog
@ -31,9 +32,15 @@ class BroadViewPublisher(BroadViewPublisherBase):
def publish(self, host, data):
code = 200
success, sdata = BSTToMonasca().serialize(host, data)
sdata = json.loads(sdata)
if self.isBST(data):
success, sdata = BSTToMonasca().serialize(host, data)
elif self.isPT(data):
success, sdata = PTToMonasca().serialize(host, data)
else:
success = False
if success:
sdata = json.loads(sdata)
for x in sdata:
syslog.syslog(json.dumps(x))
else:

1005
broadview_collector/serializers/pt_to_monasca.py Normal file
View File

File diff suppressed because it is too large Load Diff

155
broadview_collector/tools/pt/pt_drop_counter_report.py Normal file
View File

@ -0,0 +1,155 @@
# (C) Copyright Broadcom Corporation 2016
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import getopt
import sys
from pt_sim import PTSim
'''
Program to simulate sending a get-packet-trace-profile message
from a BroadView agent.
'''
class PTDropCounterReport(PTSim):
def __init__(self, host, port):
super(PTDropCounterReport, self).__init__(host, port)
self._data = {
"jsonrpc": "2.0",
"method": " get-packet-trace-drop-counter-report",
"asic-id": "1",
"version": "1",
"report": [
{
"realm": "vlan-xlate-miss-drop",
"data": [
{
"port": "1",
"count": 10
},
{
"port": "5",
"count": 20
},
{
"port": "6",
"count": 30
},
{
"port": "10",
"count": 40
},
{
"port": "11",
"count": 50
},
{
"port": "12",
"count": 60
}
]
},
{
"realm": "bpdu-drop",
"data": [
{
"port": "11",
"count": 700
},
{
"port": "15",
"count": 200
},
{
"port": "16",
"count": 300
},
{
"port": "20",
"count": 400
},
{
"port": "21",
"count": 800
},
{
"port": "22",
"count": 900
}
]
},
{
"realm": "trill-slowpath-drop",
"data": [
{
"port": "51",
"count": 310
},
{
"port": "55",
"count": 320
},
{
"port": "56",
"count": 330
},
{
"port": "60",
"count": 340
},
{
"port": "61",
"count": 350
},
{
"port": "62",
"count": 360
}
]
}
]
}
def usage():
print "pt_drop_counter_report [-h host] [-p port]"
def main():
host = None
port = None
try:
opts, args = getopt.getopt(sys.argv[1:], "h:p:")
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
for o, a in opts:
if o == "-h":
host = a
elif o == "-p":
port = a
else:
assert False, "unhandled option"
x = PTDropCounterReport(host, port)
x.send()
if __name__ == "__main__":
main()

94
broadview_collector/tools/pt/pt_drop_reason.py Normal file
View File

@ -0,0 +1,94 @@
# (C) Copyright Broadcom Corporation 2016
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import getopt
import sys
from pt_sim import PTSim
'''
Program to simulate sending a get-packet-trace-profile message
from a BroadView agent.
'''
class PTDropReason(PTSim):
def __init__(self, host, port):
super(PTDropReason, self).__init__(host, port)
self._data = {
"jsonrpc": "2.0",
"method": "get-packet-trace-drop-reason",
"asic-id": "1",
"version": "1",
"result": [
{
"reason": "l2-lookup-failure",
"port-list": [
"1",
"5",
"6",
"10-15"
],
"send-dropped-packet": 1,
"trace-profile": 0,
"packet-count": 3,
"packet-threshold": 0
},
{
"reason": "vlan-mismatch",
"port-list": [
"2",
"10",
"12",
"20-30"
],
"send-dropped-packet": 1,
"trace-profile": 1,
"packet-count": 6,
"packet-threshold": 10
}
],
"id": 1
}
def usage():
print "pt_drop_reason [-h host] [-p port]"
def main():
host = None
port = None
try:
opts, args = getopt.getopt(sys.argv[1:], "h:p:")
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
for o, a in opts:
if o == "-h":
host = a
elif o == "-p":
port = a
else:
assert False, "unhandled option"
x = PTDropReason(host, port)
x.send()
if __name__ == "__main__":
main()

106
broadview_collector/tools/pt/pt_ecmp_resolution.py Normal file
View File

@ -0,0 +1,106 @@
# (C) Copyright Broadcom Corporation 2016
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import getopt
import sys
from pt_sim import PTSim
'''
Program to simulate a get-packet-trace-ecmp-resolution message received
from a BroadView agent.
'''
class PTECMPResolution(PTSim):
def __init__(self, host, port):
super(PTECMPResolution, self).__init__(host, port)
self._data = {
"jsonrpc": "2.0",
"method": " get-packet-trace-ecmp-resolution",
"asic-id": "1",
"version": "1",
"time-stamp": self._timestamp,
"report": [
{
"port": "1",
"ecmp-link-resolution": [
{
"ecmp-group-id": "200256",
"ecmp-members": [["100004", "2.2.2.2", "28"],["100005", "6.6.6.1", "41"]],
"ecmp-dst-member": "100005",
"ecmp-dst-port": "41",
"ecmp-next-hop-ip": "6.6.6.2"
},
{
"ecmp-group-id": "200100",
"ecmp-members": [["100001", "3.3.3.1", "31"], ["100002", "7.7.7.2", "21"]],
"ecmp-dst-member": "100001",
"ecmp-dst-port": "31",
"ecmp-next-hop-ip": "3.3.3.2"
}
]
},
{
"port": "2",
"ecmp-link-resolution": [
{
"ecmp-group-id": "200512",
"ecmp-members": [["100002", "6.3.3.1", "61"], ["100004", "9.9.9.2", "41"]],
"ecmp-dst-member": "100010",
"ecmp-dst-port": "81",
"ecmp-next-hop-ip": "7.6.6.2"
},
{
"ecmp-group-id": "200200",
"ecmp-members": [["100008", "4.4.4.4", "56"],["100010", "8.8.8.1", "82"]],
"ecmp-dst-member": "100002",
"ecmp-dst-port": "62",
"ecmp-next-hop-ip": "6.5.4.3"
}
]
}
],
"id": 1
}
def usage():
print "pt_ecmp_resolution [-h host] [-p port]"
def main():
host = None
port = None
try:
opts, args = getopt.getopt(sys.argv[1:], "h:p:")
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit(2)
for o, a in opts:
if o == "-h":
host = a
elif o == "-p":
port = a
else:
assert False, "unhandled option"
x = PTECMPResolution(host, port)
x.send()
if __name__ == "__main__":
main()

95
broadview_collector/tools/pt/pt_lag_resolution.py Normal file
View File

@ -0,0 +1,95 @@
# (C) Copyright Broadcom Corporation 2016
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import getopt
import sys
from pt_sim import PTSim
'''
Program to simulate sending a get-packet-trace-lag-resolution message
from a BroadView agent.
'''
class PTLAGResolution(PTSim):
def __init__(self, host, port):
super(PTLAGResolution, self).__init__(host, port)
self._data = {
"jsonrpc": "2.0",
"method": " get-packet-trace-lag-resolution",
"asic-id": "1",
"version": "1",
"time-stamp": self._timestamp,
"report": [
{
"port": "1",
"lag-link-resolution": {
"lag-id": "1",
"lag-members": [
"1",
"2",
"3",
"4"
],
"dst-lag-member": "4"
}
},
{
"port": "2",
"lag-link-resolution": {
"lag-id": "2",
"lag-members": [
"5",
"6",
"7",
"8"
],
"dst-lag-member": "7"
}
}
],
"id": 1
}
def usage():
print "pt_lag_resolution [-h host] [-p port]"
def main():
host = None
port = None
try:
opts, args = getopt.getopt(sys.argv[1:], "h:p:")
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
for o, a in opts:
if o == "-h":
host = a
elif o == "-p":
port = a
else:
assert False, "unhandled option"
x = PTLAGResolution(host, port)
x.send()
if __name__ == "__main__":
main()

134
broadview_collector/tools/pt/pt_profile.py Normal file
View File

@ -0,0 +1,134 @@
# (C) Copyright Broadcom Corporation 2016
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import getopt
import sys
from pt_sim import PTSim
'''
Program to simulate sending a get-packet-trace-profile message
from a BroadView agent.
'''
class PTProfile(PTSim):
def __init__(self, host, port):
super(PTProfile, self).__init__(host, port)
self._data = {
"jsonrpc": "2.0",
"method": "get-packet-trace-profile",
"asic-id": "1",
"version": "1",
"time-stamp": "2014-11-18 - 00:15:04 ",
"report": [
{
"port": "1",
"trace-profile": [
{
"realm": "lag-link-resolution",
"data": {
"lag-id": "2",
"lag-members": ["1", "2", "3", "4"],
"dst-lag-member": "4"
}
},
{
"realm": "ecmp-link-resolution",
"data": [
{
"ecmp-group-id": "200256",
"ecmp-members": [["100004", "2.2.2.2", "28"],["100005", "6.6.6.1", "41"]],
"ecmp-dst-member": "100005",
"ecmp-dst-port": "41",
"ecmp-next-hop-ip": "6.6.6.2"
},
{
"ecmp-group-id": "200100",
"ecmp-members": [["100001", "3.3.3.1", "31"],
["100002", "7.7.7.2", "21"]],
"ecmp-dst-member": "100001",
"ecmp-dst-port": "31",
"ecmp-next-hop-ip": "3.3.3.2"
}
]
}
]
},
{
"port": "2",
"trace-profile": [
{
"realm": "lag-link-resolution",
"data": {
"lag-id": "3",
"lag-members": ["5","6","7","8"],
"dst-lag-member": "6"
}
},
{
"realm": "ecmp-link-resolution",
"data": [
{
"ecmp-group-id": "200512",
"ecmp-members": [["200004", "3.2.2.2", "38"],["100005", "6.6.6.1", "41"]],
"ecmp-dst-member": "100010",
"ecmp-dst-port": "19",
"ecmp-next-hop-ip": "8.8.8.2"
},
{
"ecmp-group-id": "200200",
"ecmp-members": [["100002", "4.3.3.1", "76"], ["100002", "7.7.7.2", "21"]],
"ecmp-dst-member": "100002",
"ecmp-dst-port": "55",
"ecmp-next-hop-ip": "7.3.3.2"
}
]
}
]
}
],
"id": 1
}
def usage():
print "pt_profile [-h host] [-p port]"
def main():
host = None
port = None
try:
opts, args = getopt.getopt(sys.argv[1:], "h:p:")
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
for o, a in opts:
if o == "-h":
host = a
elif o == "-p":
port = a
else:
assert False, "unhandled option"
x = PTProfile(host, port)
x.send()
if __name__ == "__main__":
main()

51
broadview_collector/tools/pt/pt_sim.py Normal file
View File

@ -0,0 +1,51 @@
# (C) Copyright Broadcom Corporation 2016
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import requests
import datetime
import time
'''
Base class for packet trace agent simulation classes
'''
class PTSim(object):
def __init__(self, host=None, port=None):
if not host:
self._host = "127.0.0.1"
else:
self._host = host
if not port:
self._port = 8082
else:
self._port = port
self.setUp()
def setUp(self):
# convert datetime string to timestamp
d = str(datetime.datetime.now()).split(" ")
t = d[1].split(".")[0]
self._timestamp = "{} - {}".format(d[0], t)
# initialize data, overridden by subclass
self._data = None
def send(self):
j = json.dumps(self._data)
r = requests.post('http://{}:{}'.format(self._host, self._port), json=j)