Replace fixed endpoint config by dynamic in devstack plugin

Some of the fixed endpoint configs caused errors.
Magnum and Zun explicitly call for grabbing the API endpoint from
catalog. Switch all the JSON ingester configs to use API endpoint
from service catalog to avoid incorrect config.

Change-Id: I50d2755317f68928f41d3770e25dfe67ba088969
Closes-bug: 1819556
This commit is contained in:
Eric K 2019-03-11 16:57:58 -07:00
parent e84057a6f5
commit ea1fb9c3ed
17 changed files with 136 additions and 116 deletions

View File

@ -65,15 +65,7 @@ class ExecApiManager(object):
'supported.'.auth_config['type'])
name = config['name']
port = config.get('api_endpoint_port')
if port is not None:
port_str = ':' + str(port)
else:
port_str = ''
self._exec_api_endpoints[name] = (
config.get('api_endpoint_host', '').rstrip('/')
+ port_str + '/'
+ config.get('api_endpoint_path', '').lstrip('/'))
self._exec_api_endpoints[name] = config['api_endpoint']
self._exec_api_sessions[name] = session
@lockutils.synchronized('congress_json_ingester_exec_api')

View File

@ -74,15 +74,7 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver):
self._create_schema_and_tables()
self.poll_time = self._config.get('poll_interval', 60)
self._setup_table_key_sets()
port = self._config.get('api_endpoint_port')
if port is not None:
port_str = ':' + str(port)
else:
port_str = ''
self._api_endpoint = (
self._config.get('api_endpoint_host', '').rstrip('/')
+ port_str + '/'
+ self._config.get('api_endpoint_path', '').lstrip('/'))
self._api_endpoint = self._config.get('api_endpoint')
self._initialize_session()
self._initialize_update_methods()
if len(self.update_methods) > 0:
@ -235,8 +227,9 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver):
else:
LOG.error('authentication type %s not supported.',
auth_config.get['type'])
raise exception.BadConfig('authentication type {} not '
'supported.'.auth_config['type'])
raise exception.BadConfig(
'authentication type {} not supported.'.format(
auth_config['type']))
def _initialize_update_methods(self):
for table_name in self._config['tables']:
@ -247,17 +240,23 @@ class JsonIngester(datasource_driver.PollingDataSourceDriver):
# variables in closure
def update_method(
table_name=table_name, table_info=table_info):
full_path = self._api_endpoint.rstrip(
'/') + '/' + table_info['api_path'].lstrip('/')
result = self._session.get(full_path).json()
# FIXME: generalize to other verbs?
try:
full_path = self._api_endpoint.rstrip(
'/') + '/' + table_info['api_path'].lstrip('/')
result = self._session.get(full_path).json()
# FIXME: generalize to other verbs?
jsonpath_expr = parser.parse(table_info['jsonpath'])
ingest_data = [match.value for match in
jsonpath_expr.find(result)]
self.state[table_name] = set(
[json.dumps(item, sort_keys=True)
for item in ingest_data])
jsonpath_expr = parser.parse(table_info['jsonpath'])
ingest_data = [match.value for match in
jsonpath_expr.find(result)]
self.state[table_name] = set(
[json.dumps(item, sort_keys=True)
for item in ingest_data])
except BaseException:
LOG.exception('Exception occurred while updating '
'table %s.%s from: URL %s',
self.name, table_name,
full_path)
self.add_update_method(update_method, table_name)

View File

@ -38,8 +38,7 @@ class TestExecApiManager(base.TestCase):
self.test_configs = [
{
"api_endpoint_host": "test1",
"api_endpoint_path": "url",
"api_endpoint": "test1/url",
"tables": {
"flavors": {
"poll": {
@ -75,8 +74,7 @@ class TestExecApiManager(base.TestCase):
},
{
"allow_exec_api": True,
"api_endpoint_host": "test2",
"api_endpoint_path": "url",
"api_endpoint": "test2/url",
"tables": {
"flavors": {
"poll": {
@ -112,8 +110,7 @@ class TestExecApiManager(base.TestCase):
},
{
"allow_exec_api": True,
"api_endpoint_host": "test3",
"api_endpoint_path": "url",
"api_endpoint": "test3/url",
"authentication": {
"type": "keystone",
"config": {

View File

@ -70,28 +70,60 @@ function configure_congress {
iniset $CONGRESS_CONF json_ingester db_connection ${db_connection_mysql/?*:\/\//postgresql:\/\/}
iniset $CONGRESS_CONF json_ingester config_path "$CONGRESS_JSON_CONF_DIR"
iniset $CONGRESS_CONF json_ingester config_reusables_path "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo "primary_host: http://$SERVICE_HOST" > "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo "keystone_admin_auth_config:" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
# setup json ingester config files
if [[ ! -d $CONGRESS_JSON_CONF_DIR ]]; then
mkdir $CONGRESS_JSON_CONF_DIR
fi
echo "keystone_admin_auth_config:" > "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " type: keystone" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " config:" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " project_name: $OS_PROJECT_NAME" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " username: $OS_USERNAME" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " password: $OS_PASSWORD" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
echo " auth_url: http://$SERVICE_HOST/identity" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
local OS_PROJECT_ID="$(openstack project show $OS_PROJECT_NAME -f value -c id)"
echo "cinder_path: volume/v3/$OS_PROJECT_ID/"
echo "heat_path: orchestration/v1/$OS_PROJECT_ID/"
if [[ ! -d $CONGRESS_JSON_CONF_DIR ]]; then
mkdir $CONGRESS_JSON_CONF_DIR
fi
cp -r $CONGRESS_DIR/etc/sample_json_ingesters/* $CONGRESS_JSON_CONF_DIR
echo " auth_url: `openstack catalog show keystone -f value -c endpoints | sed -n 's/^\s*public: //p' | sed -n '1p'`" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
fi
_congress_setup_keystone $CONGRESS_CONF keystone_authtoken
}
function _configure_congress_json_ingester {
local endpoint=`openstack catalog show $1 -f value -c endpoints | sed -n 's/^\s*public:\s*//p' | sed -n '1p'`
if [[ ! -z $endpoint ]]; then
echo "$1_api_endpoint: $endpoint" >> "$CONGRESS_JSON_CONF_REUSABLES_PATH"
_copy_congress_json_ingester_config $1
fi
}
function _copy_congress_json_ingester_config {
cp $CONGRESS_DIR/etc/sample_json_ingesters/$1.yaml $CONGRESS_JSON_CONF_DIR/
}
function configure_congress_datasources {
if [ "$ENABLE_CONGRESS_JSON" == "True" ]; then
_configure_congress_json_ingester cinderv3
_configure_congress_json_ingester glance
_configure_congress_json_ingester heat
_configure_congress_json_ingester keystone
_configure_congress_json_ingester magnum
_configure_congress_json_ingester masakari
_configure_congress_json_ingester mistral
_configure_congress_json_ingester neutron
_configure_congress_json_ingester nova
_configure_congress_json_ingester tacker
_configure_congress_json_ingester zun
_copy_congress_json_ingester_config monasca
_copy_congress_json_ingester_config cve
if [ "$CONGRESS_MULTIPROCESS_DEPLOYMENT" == "False" ]; then
restart_service devstack@congress.service
echo "Waiting for Congress to restart..."
if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget --no-proxy -q -O- http://$CONGRESS_HOST:$CONGRESS_PORT; do sleep 1; done"; then
die $LINENO "Congress did not restart"
fi
else
restart_service devstack@congress-datasources.service
fi
fi
_configure_service neutron neutronv2
_configure_service neutron-qos neutronv2_qos
_configure_service nova nova
@ -343,6 +375,13 @@ function start_congress_service_and_check {
fi
}
function _wait_for_congress {
echo "Waiting for Congress to start..."
if ! timeout $SERVICE_TIMEOUT sh -c "while ! wget --no-proxy -q -O- http://$CONGRESS_HOST:$CONGRESS_PORT; do sleep 1; done"; then
die $LINENO "Congress did not start"
fi
}
# stop_congress() - Stop running processes (non-screen)
function stop_congress {

View File

@ -2,8 +2,7 @@ name: _volume
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: !ref cinder_path
api_endpoint: !ref cinderv3_api_endpoint
tables:
volumes:
poll:

View File

@ -1,8 +1,7 @@
name: _cve
allow_exec_api: true
poll_interval: 86400 # one day
api_endpoint_host: https://cve.circl.lu
api_endpoint_path: api/
api_endpoint: https://cve.circl.lu/api/
tables:
linux_kernel:
poll:

View File

@ -2,8 +2,7 @@ name: _image
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v2/
api_endpoint: !ref glance_api_endpoint
tables:
images:
poll:

View File

@ -2,8 +2,7 @@ name: _heat
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: !ref heat_path
api_endpoint: !ref heat_api_endpoint
tables:
stacks:
poll:

View File

@ -2,53 +2,50 @@ name: _identity
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: identity/v3/
api_default_headers:
X-OpenStack-Nova-API-Version: "2.26"
api_endpoint: !ref keystone_api_endpoint
tables:
domains:
poll:
api_path: domains
api_path: v3/domains
api_method: get
jsonpath: $.domains[:]
groups:
poll:
api_path: groups
api_path: v3/groups
api_method: get
jsonpath: $.groups[:]
projects:
poll:
api_path: projects
api_path: v3/projects
api_method: get
jsonpath: $.projects[:]
regions:
poll:
api_path: regions
api_path: v3/regions
api_method: get
jsonpath: $.regions[:]
roles:
poll:
api_path: roles
api_path: v3/roles
api_method: get
jsonpath: $.roles[:]
services:
poll:
api_path: services
api_path: v3/services
api_method: get
jsonpath: $.services[:]
endpoints:
poll:
api_path: endpoints
api_path: v3/endpoints
api_method: get
jsonpath: $.endpoints[:]
registered_limits:
poll:
api_path: registered_limits
api_path: v3/registered_limits
api_method: get
jsonpath: $.registered_limits[:]
users:
poll:
api_path: users
api_path: v3/users
api_method: get
jsonpath: $.users[:]

View File

@ -2,8 +2,7 @@ name: _magnum
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: magnum/v1/
api_endpoint: !ref magnum_api_endpoint
tables:
bays:
poll:

View File

@ -2,9 +2,7 @@ name: _masakari
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v1/
api_endpoint_port: 15868
api_endpoint: !ref masakari_api_endpoint
tables:
segments:
poll:

View File

@ -2,9 +2,7 @@ name: _mistral
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v2/
api_endpoint_port: 8989
api_endpoint: !ref mistral_api_endpoint
tables:
workbooks:
poll:
@ -46,8 +44,11 @@ tables:
api_path: environments
api_method: get
jsonpath: $.environments[:]
services:
poll:
api_path: services
api_method: get
jsonpath: $.services[:]
# This API is not supported by default.
# uncomment if available
# see: https://docs.openstack.org/mistral/latest/api/v2.html#services
# services:
# poll:
# api_path: services
# api_method: get
# jsonpath: $.services[:]

View File

@ -0,0 +1,21 @@
name: _fwaas
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint: !ref neutron_api_endpoint
tables:
firewall_groups:
poll:
api_path: v2.0/fwaas/firewall_groups
api_method: get
jsonpath: $.firewall_groups[:]
firewall_policies:
poll:
api_path: v2.0/fwaas/firewall_policies
api_method: get
jsonpath: $.firewall_policies[:]
firewall_rules:
poll:
api_path: v2.0/fwaas/firewall_rules
api_method: get
jsonpath: $.firewall_rules[:]

View File

@ -2,62 +2,45 @@ name: _networking
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v2.0/
api_endpoint_port: 9696
api_endpoint: !ref neutron_api_endpoint
tables:
networks:
poll:
api_path: networks
api_path: v2.0/networks
api_method: get
jsonpath: $.networks[:]
ports:
poll:
api_path: ports
api_path: v2.0/ports
api_method: get
jsonpath: $.ports[:]
floatingips:
poll:
api_path: floatingips
api_path: v2.0/floatingips
api_method: get
jsonpath: $.floatingips[:]
routers:
poll:
api_path: routers
api_path: v2.0/routers
api_method: get
jsonpath: $.routers[:]
subnets:
poll:
api_path: subnets
api_path: v2.0/subnets
api_method: get
jsonpath: $.subnets[:]
firewall_groups:
poll:
api_path: fwaas/firewall_groups
api_method: get
jsonpath: $.firewall_groups[:]
firewall_policies:
poll:
api_path: fwaas/firewall_policies
api_method: get
jsonpath: $.firewall_policies[:]
firewall_rules:
poll:
api_path: fwaas/firewall_rules
api_method: get
jsonpath: $.firewall_rules[:]
security_groups:
poll:
api_path: security-groups
api_path: v2.0/security-groups
api_method: get
jsonpath: $.security_groups[:]
rbac_policies:
poll:
api_path: rbac-policies
api_path: v2.0/rbac-policies
api_method: get
jsonpath: $.rbac_policies[:]
agents:
poll:
api_path: agents
api_path: v2.0/agents
api_method: get
jsonpath: $.agents[:]

View File

@ -2,8 +2,7 @@ name: _compute
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: compute/v2.1/
api_endpoint: !ref nova_api_endpoint
api_default_headers:
X-OpenStack-Nova-API-Version: "2.26"
tables:

View File

@ -2,17 +2,15 @@ name: _tacker
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: v1.0/
api_endpoint_port: 9890
api_endpoint: !ref tacker_api_endpoint
tables:
vnfds:
poll:
api_path: vnfds
api_path: v1.0/vnfds
api_method: get
jsonpath: $.vnfds[:]
vnfs:
poll:
api_path: vnfs
api_path: v1.0/vnfs
api_method: get
jsonpath: $.vnfs[:]

View File

@ -2,8 +2,9 @@ name: _zun
poll_interval: 60
allow_exec_api: true
authentication: !ref keystone_admin_auth_config
api_endpoint_host: !ref primary_host
api_endpoint_path: zun/v1/
api_endpoint: !ref zun_api_endpoint
api_default_headers:
OpenStack-API-Version: container 1.4
tables:
containers:
poll: