Merge "Replace fixed endpoint config by dynamic in devstack plugin"

This commit is contained in:
Zuul 2019-03-19 19:30:20 +00:00 committed by Gerrit Code Review
commit c1a9d2a095
17 changed files with 136 additions and 116 deletions

View File

@ -65,15 +65,7 @@ class ExecApiManager(object):
'supported.'.auth_config['type'])
name = config['name']
port = config.get('api_endpoint_port')
if port is not None:
port_str = ':' + str(port)
else:
port_str = ''
self._exec_api_endpoints[name] = (
config.get('api_endpoint_host', '').rstrip('/')
+ port_str + '/'
+ config.get('api_endpoint_path', '').lstrip('/'))
self._exec_api_endpoints[name] = config['api_endpoint']
self._exec_api_sessions[name] = session
@lockutils.synchronized('congress_json_ingester_exec_api')

View File

@ -74,15 +74,7 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver):
self._create_schema_and_tables()
self.poll_time = self._config.get('poll_interval', 60)
self._setup_table_key_sets()
port = self._config.get('api_endpoint_port')
if port is not None:
port_str = ':' + str(port)
else:
port_str = ''
self._api_endpoint = (
self._config.get('api_endpoint_host', '').rstrip('/')
+ port_str + '/'
+ self._config.get('api_endpoint_path', '').lstrip('/'))
self._api_endpoint = self._config.get('api_endpoint')
self._initialize_session()
self._initialize_update_methods()
if len(self.update_methods) > 0:
@ -251,8 +243,9 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver):
else:
LOG.error('authentication type %s not supported.',
auth_config.get['type'])
raise exception.BadConfig('authentication type {} not '
'supported.'.auth_config['type'])
raise exception.BadConfig(
'authentication type {} not supported.'.format(
auth_config['type']))
def _initialize_update_methods(self):
for table_name in self._config['tables']:
@ -263,17 +256,23 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver):
# variables in closure
def update_method(
table_name=table_name, table_info=table_info):
full_path = self._api_endpoint.rstrip(
'/') + '/' + table_info['api_path'].lstrip('/')
result = self._session.get(full_path).json()
# FIXME: generalize to other verbs?
try:
full_path = self._api_endpoint.rstrip(
'/') + '/' + table_info['api_path'].lstrip('/')
result = self._session.get(full_path).json()
# FIXME: generalize to other verbs?
jsonpath_expr = parser.parse(table_info['jsonpath'])
ingest_data = [match.value for match in
jsonpath_expr.find(result)]
self.state[table_name] = set(
[json.dumps(item, sort_keys=True)
for item in ingest_data])
jsonpath_expr = parser.parse(table_info['jsonpath'])
ingest_data = [match.value for match in
jsonpath_expr.find(result)]
self.state[table_name] = set(
[json.dumps(item, sort_keys=True)
for item in ingest_data])
except BaseException:
LOG.exception('Exception occurred while updating '
'table %s.%s from: URL %s',
self.name, table_name,
full_path)
self.add_update_method(update_method, table_name)

View File

@ -38,8 +38,7 @@ class TestExecApiManager(base.TestCase):
self.test_configs = [
{
"api_endpoint_host": "test1",
"api_endpoint_path": "url",
"api_endpoint": "test1/url",
"tables": {
"flavors": {
"poll": {
@ -75,8 +74,7 @@ class TestExecApiManager(base.TestCase):
},
{
"allow_exec_api": True,
"api_endpoint_host": "test2",
"api_endpoint_path": "url",
"api_endpoint": "test2/url",
"tables": {
"flavors": {
"poll": {
@ -112,8 +110,7 @@ class TestExecApiManager(base.TestCase):
},
{
"allow_exec_api": True,
"api_endpoint_host": "test3",
"api_endpoint_path": "url",
"api_endpoint": "test3/url",
"authentication": {
"type": "keystone",
"config": {

View File

@ -70,28 +70,60 @@ function configure_congress {
iniset $CONGRESS_CONF json_ingester db_connection ${db_connection_mysql/?*:\/\//postgresql:\/\/}
iniset $CONGRESS_CONF json_ingester config_path "$CONGRESS_JSON_CONF_DIR"
iniset $CONGRESS_CONF json_ingester config_reusables_path "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo "primary_host: http://$SERVICE_HOST" > "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo "keystone_admin_auth_config:" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
# setup json ingester config files
if [[ ! -d $CONGRESS_JSON_CONF_DIR ]]; then
mkdir $CONGRESS_JSON_CONF_DIR
fi
echo "keystone_admin_auth_config:" > "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " type: keystone" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " config:" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " project_name: $OS_PROJECT_NAME" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " username: $OS_USERNAME" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " password: $OS_PASSWORD" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " auth_url: http://$SERVICE_HOST/identity" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
local OS_PROJECT_ID="$(openstack project show $OS_PROJECT_NAME -f value -c id)"
echo "cinder_path: volume/v3/$OS_PROJECT_ID/"
echo "heat_path: orchestration/v1/$OS_PROJECT_ID/"
if [[ ! -d $CONGRESS_JSON_CONF_DIR ]]; then
mkdir $CONGRESS_JSON_CONF_DIR
fi
cp -r $CONGRESS_DIR/etc/sample_json_ingesters/* $CONGRESS_JSON_CONF_DIR
echo " auth_url: `openstack catalog show keystone -f value -c endpoints | sed -n 's/^\s*public: //p' | sed -n '1p'`" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
fi
_congress_setup_keystone $CONGRESS_CONF keystone_authtoken
}
function _configure_congress_json_ingester {
local endpoint=`openstack catalog show $1 -f value -c endpoints | sed -n 's/^\s*public:\s*//p' | sed -n '1p'`
if [[ ! -z $endpoint ]]; then
echo "$1_api_endpoint: $endpoint" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
_copy_congress_json_ingester_config $1
fi
}
function _copy_congress_json_ingester_config {
cp $CONGRESS_DIR/etc/sample_json_ingesters/$1.yaml $CONGRESS_JSON_CONF_DIR/
}
function configure_congress_datasources {
if [ "$ENABLE_CONGRESS_JSON" == "True" ]; then
_configure_congress_json_ingester cinderv3
_configure_congress_json_ingester glance
_configure_congress_json_ingester heat
_configure_congress_json_ingester keystone
_configure_congress_json_ingester magnum
_configure_congress_json_ingester masakari
_configure_congress_json_ingester mistral
_configure_congress_json_ingester neutron
_configure_congress_json_ingester nova
_configure_congress_json_ingester tacker
_configure_congress_json_ingester zun
_copy_congress_json_ingester_config monasca
_copy_congress_json_ingester_config cve
if [ "$CONGRESS_MULTIPROCESS_DEPLOYMENT" == "False" ]; then
restart_service devstack@congress.service
echo "Waiting for Congress to restart..."
if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget --no-proxy -q -O- http://$CONGRESS_HOST:$CONGRESS_PORT; do sleep 1; done"; then
die $LINENO "Congress did not restart"
fi
else
restart_service devstack@congress-datasources.service
fi
fi
_configure_service neutron neutronv2
_configure_service neutron-qos neutronv2_qos
_configure_service nova nova
@ -343,6 +375,13 @@ function start_congress_service_and_check {
fi
}
function _wait_for_congress {
echo "Waiting for Congress to start..."
if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget --no-proxy -q -O- http://$CONGRESS_HOST:$CONGRESS_PORT; do sleep 1; done"; then
die $LINENO "Congress did not start"
fi
}
# stop_congress() - Stop running processes (non-screen)
function stop_congress {

View File

@ -2,8 +2,7 @@ name: _volume
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: !ref cinder_path
api_endpoint: !ref cinderv3_api_endpoint
tables:
volumes:
poll:

View File

@ -1,8 +1,7 @@
name: _cve
allow_exec_api: true
poll_interval: 86400 # one day
api_endpoint_host: https://cve.circl.lu
api_endpoint_path: api/
api_endpoint: https://cve.circl.lu/api/
tables:
linux_kernel:
gin_index: false

View File

@ -2,8 +2,7 @@ name: _image
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v2/
api_endpoint: !ref glance_api_endpoint
tables:
images:
poll:

View File

@ -2,8 +2,7 @@ name: _heat
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: !ref heat_path
api_endpoint: !ref heat_api_endpoint
tables:
stacks:
poll:

View File

@ -2,53 +2,50 @@ name: _identity
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: identity/v3/
api_default_headers:
X-OpenStack-Nova-API-Version: "2.26"
api_endpoint: !ref keystone_api_endpoint
tables:
domains:
poll:
api_path: domains
api_path: v3/domains
api_method: get
jsonpath: $.domains[:]
groups:
poll:
api_path: groups
api_path: v3/groups
api_method: get
jsonpath: $.groups[:]
projects:
poll:
api_path: projects
api_path: v3/projects
api_method: get
jsonpath: $.projects[:]
regions:
poll:
api_path: regions
api_path: v3/regions
api_method: get
jsonpath: $.regions[:]
roles:
poll:
api_path: roles
api_path: v3/roles
api_method: get
jsonpath: $.roles[:]
services:
poll:
api_path: services
api_path: v3/services
api_method: get
jsonpath: $.services[:]
endpoints:
poll:
api_path: endpoints
api_path: v3/endpoints
api_method: get
jsonpath: $.endpoints[:]
registered_limits:
poll:
api_path: registered_limits
api_path: v3/registered_limits
api_method: get
jsonpath: $.registered_limits[:]
users:
poll:
api_path: users
api_path: v3/users
api_method: get
jsonpath: $.users[:]

View File

@ -2,8 +2,7 @@ name: _magnum
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: magnum/v1/
api_endpoint: !ref magnum_api_endpoint
tables:
bays:
poll:

View File

@ -2,9 +2,7 @@ name: _masakari
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v1/
api_endpoint_port: 15868
api_endpoint: !ref masakari_api_endpoint
tables:
segments:
poll:

View File

@ -2,9 +2,7 @@ name: _mistral
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v2/
api_endpoint_port: 8989
api_endpoint: !ref mistral_api_endpoint
tables:
workbooks:
poll:
@ -46,8 +44,11 @@ tables:
api_path: environments
api_method: get
jsonpath: $.environments[:]
services:
poll:
api_path: services
api_method: get
jsonpath: $.services[:]
# This API is not supported by default.
# uncomment if available
# see: https://docs.openstack.org/mistral/latest/api/v2.html#services
# services:
# poll:
# api_path: services
# api_method: get
# jsonpath: $.services[:]

View File

@ -0,0 +1,21 @@
name: _fwaas
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint: !ref neutron_api_endpoint
tables:
firewall_groups:
poll:
api_path: v2.0/fwaas/firewall_groups
api_method: get
jsonpath: $.firewall_groups[:]
firewall_policies:
poll:
api_path: v2.0/fwaas/firewall_policies
api_method: get
jsonpath: $.firewall_policies[:]
firewall_rules:
poll:
api_path: v2.0/fwaas/firewall_rules
api_method: get
jsonpath: $.firewall_rules[:]

View File

@ -2,62 +2,45 @@ name: _networking
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v2.0/
api_endpoint_port: 9696
api_endpoint: !ref neutron_api_endpoint
tables:
networks:
poll:
api_path: networks
api_path: v2.0/networks
api_method: get
jsonpath: $.networks[:]
ports:
poll:
api_path: ports
api_path: v2.0/ports
api_method: get
jsonpath: $.ports[:]
floatingips:
poll:
api_path: floatingips
api_path: v2.0/floatingips
api_method: get
jsonpath: $.floatingips[:]
routers:
poll:
api_path: routers
api_path: v2.0/routers
api_method: get
jsonpath: $.routers[:]
subnets:
poll:
api_path: subnets
api_path: v2.0/subnets
api_method: get
jsonpath: $.subnets[:]
firewall_groups:
poll:
api_path: fwaas/firewall_groups
api_method: get
jsonpath: $.firewall_groups[:]
firewall_policies:
poll:
api_path: fwaas/firewall_policies
api_method: get
jsonpath: $.firewall_policies[:]
firewall_rules:
poll:
api_path: fwaas/firewall_rules
api_method: get
jsonpath: $.firewall_rules[:]
security_groups:
poll:
api_path: security-groups
api_path: v2.0/security-groups
api_method: get
jsonpath: $.security_groups[:]
rbac_policies:
poll:
api_path: rbac-policies
api_path: v2.0/rbac-policies
api_method: get
jsonpath: $.rbac_policies[:]
agents:
poll:
api_path: agents
api_path: v2.0/agents
api_method: get
jsonpath: $.agents[:]

View File

@ -2,8 +2,7 @@ name: _compute
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: compute/v2.1/
api_endpoint: !ref nova_api_endpoint
api_default_headers:
X-OpenStack-Nova-API-Version: "2.26"
tables:

View File

@ -2,17 +2,15 @@ name: _tacker
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v1.0/
api_endpoint_port: 9890
api_endpoint: !ref tacker_api_endpoint
tables:
vnfds:
poll:
api_path: vnfds
api_path: v1.0/vnfds
api_method: get
jsonpath: $.vnfds[:]
vnfs:
poll:
api_path: vnfs
api_path: v1.0/vnfs
api_method: get
jsonpath: $.vnfs[:]

View File

@ -2,8 +2,9 @@ name: _zun
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: zun/v1/
api_endpoint: !ref zun_api_endpoint
api_default_headers:
OpenStack-API-Version: container 1.4
tables:
containers:
poll: