From 7676be3cb64af628b5736c2b4e48a59e6236b479 Mon Sep 17 00:00:00 2001 From: Eric Kao Date: Wed, 6 Mar 2019 18:59:08 -0800 Subject: [PATCH] Allow port in JSON ingester config Allow config to specify a port to use in making the API calls to ingest JSON. Also added additional sample configs: neutron, keystone, monasca. Change-Id: I7d852742d0943c3857ee4e04d5a0d77b6bafeed7 partially-implements: bp json-data-model --- .../datasources/json_ingester/exec_api.py | 8 ++- .../json_ingester/json_ingester.py | 11 +++- etc/sample_json_ingesters/cve.yaml | 11 ++++ etc/sample_json_ingesters/keystone.yaml | 54 ++++++++++++++++ etc/sample_json_ingesters/monasca.yaml | 6 ++ etc/sample_json_ingesters/neutron.yaml | 63 +++++++++++++++++++ etc/sample_json_ingesters/nova.yaml | 28 +++++++-- 7 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 etc/sample_json_ingesters/cve.yaml create mode 100644 etc/sample_json_ingesters/keystone.yaml create mode 100644 etc/sample_json_ingesters/monasca.yaml create mode 100644 etc/sample_json_ingesters/neutron.yaml diff --git a/congress/datasources/json_ingester/exec_api.py b/congress/datasources/json_ingester/exec_api.py index a82c416ae..aeed7afc2 100644 --- a/congress/datasources/json_ingester/exec_api.py +++ b/congress/datasources/json_ingester/exec_api.py @@ -65,8 +65,14 @@ class ExecApiManager(object): 'supported.'.auth_config['type']) name = config['name'] + port = config.get('api_endpoint_port') + if port is not None: + port_str = ':' + str(port) + else: + port_str = '' self._exec_api_endpoints[name] = ( - config.get('api_endpoint_host', '').rstrip('/') + '/' + config.get('api_endpoint_host', '').rstrip('/') + + port_str + '/' + config.get('api_endpoint_path', '').lstrip('/')) self._exec_api_sessions[name] = session diff --git a/congress/datasources/json_ingester/json_ingester.py b/congress/datasources/json_ingester/json_ingester.py index 3bd0f506b..4c1559184 100644 --- a/congress/datasources/json_ingester/json_ingester.py +++ b/congress/datasources/json_ingester/json_ingester.py @@ -74,8 +74,15 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver): self._create_schema_and_tables() self.poll_time = self._config.get('poll_interval', 60) self._setup_table_key_sets() - self._api_endpoint = self._config.get('api_endpoint_host', '').rstrip( - '/') + '/' + self._config.get('api_endpoint_path', '').lstrip('/') + port = self._config.get('api_endpoint_port') + if port is not None: + port_str = ':' + str(port) + else: + port_str = '' + self._api_endpoint = ( + self._config.get('api_endpoint_host', '').rstrip('/') + + port_str + '/' + + self._config.get('api_endpoint_path', '').lstrip('/')) self._initialize_session() self._initialize_update_methods() if len(self.update_methods) > 0: diff --git a/etc/sample_json_ingesters/cve.yaml b/etc/sample_json_ingesters/cve.yaml new file mode 100644 index 000000000..e712c1124 --- /dev/null +++ b/etc/sample_json_ingesters/cve.yaml @@ -0,0 +1,11 @@ +name: _cve +allow_exec_api: true +poll_interval: 86400 # one day +api_endpoint_host: https://cve.circl.lu +api_endpoint_path: api/ +tables: + linux_kernel: + poll: + api_path: search/linux/kernel + api_method: get + jsonpath: $[:] diff --git a/etc/sample_json_ingesters/keystone.yaml b/etc/sample_json_ingesters/keystone.yaml new file mode 100644 index 000000000..1251aaa19 --- /dev/null +++ b/etc/sample_json_ingesters/keystone.yaml @@ -0,0 +1,54 @@ +name: _identity +poll_interval: 60 +allow_exec_api: true +authentication: !ref keystone_admin_auth_config +api_endpoint_host: !ref primary_host +api_endpoint_path: identity/v3/ +api_default_headers: + X-OpenStack-Nova-API-Version: "2.26" +tables: + domains: + poll: + api_path: domains + api_method: get + jsonpath: $.domains[:] + groups: + poll: + api_path: groups + api_method: get + jsonpath: $.groups[:] + projects: + poll: + api_path: projects + api_method: get + jsonpath: $.projects[:] + regions: + poll: + api_path: regions + api_method: get + jsonpath: $.regions[:] + roles: + poll: + api_path: roles + api_method: get + jsonpath: $.roles[:] + services: + poll: + api_path: services + api_method: get + jsonpath: $.services[:] + endpoints: + poll: + api_path: endpoints + api_method: get + jsonpath: $.endpoints[:] + registered_limits: + poll: + api_path: registered_limits + api_method: get + jsonpath: $.registered_limits[:] + users: + poll: + api_path: users + api_method: get + jsonpath: $.users[:] diff --git a/etc/sample_json_ingesters/monasca.yaml b/etc/sample_json_ingesters/monasca.yaml new file mode 100644 index 000000000..98e9b4c81 --- /dev/null +++ b/etc/sample_json_ingesters/monasca.yaml @@ -0,0 +1,6 @@ +name: _monasca +tables: + alarms: + webhook: + record_jsonpath: $ + id_jsonpath: $.alarm_id diff --git a/etc/sample_json_ingesters/neutron.yaml b/etc/sample_json_ingesters/neutron.yaml new file mode 100644 index 000000000..1b7d0b5fb --- /dev/null +++ b/etc/sample_json_ingesters/neutron.yaml @@ -0,0 +1,63 @@ +name: _networking +poll_interval: 60 +allow_exec_api: true +authentication: !ref keystone_admin_auth_config +api_endpoint_host: !ref primary_host +api_endpoint_path: v2.0/ +api_endpoint_port: 9696 +tables: + networks: + poll: + api_path: networks + api_method: get + jsonpath: $.networks[:] + ports: + poll: + api_path: ports + api_method: get + jsonpath: $.ports[:] + floatingips: + poll: + api_path: floatingips + api_method: get + jsonpath: $.floatingips[:] + routers: + poll: + api_path: routers + api_method: get + jsonpath: $.routers[:] + subnets: + poll: + api_path: subnets + api_method: get + jsonpath: $.subnets[:] + firewall_groups: + poll: + api_path: fwaas/firewall_groups + api_method: get + jsonpath: $.firewall_groups[:] + firewall_policies: + poll: + api_path: fwaas/firewall_policies + api_method: get + jsonpath: $.firewall_policies[:] + firewall_rules: + poll: + api_path: fwaas/firewall_rules + api_method: get + jsonpath: $.firewall_rules[:] + security_groups: + poll: + api_path: security-groups + api_method: get + jsonpath: $.security_groups[:] + rbac_policies: + poll: + api_path: rbac-policies + api_method: get + jsonpath: $.rbac_policies[:] + agents: + poll: + api_path: agents + api_method: get + jsonpath: $.agents[:] diff --git a/etc/sample_json_ingesters/nova.yaml b/etc/sample_json_ingesters/nova.yaml index f3e951ca9..4c3304d7f 100644 --- a/etc/sample_json_ingesters/nova.yaml +++ b/etc/sample_json_ingesters/nova.yaml @@ -1,5 +1,5 @@ -name: nova -poll_interval: 5 +name: _compute +poll_interval: 60 allow_exec_api: true authentication: !ref keystone_admin_auth_config api_endpoint_host: !ref primary_host @@ -17,7 +17,23 @@ tables: api_path: servers/detail api_method: get jsonpath: $.servers[:] - alarms: - webhook: - record_jsonpath: $.payload - id_jsonpath: $.id + hypervisors: + poll: + api_path: os-hypervisors/detail + api_method: get + jsonpath: $.hypervisors[:] + availability_zones: + poll: + api_path: os-availability-zone/detail + api_method: get + jsonpath: $.availabilityZoneInfo[:] + migrations: + poll: + api_path: os-migrations + api_method: get + jsonpath: $.migrations[:] + services: + poll: + api_path: os-services + api_method: get + jsonpath: $.services[:]