367 lines
11 KiB
Python
367 lines
11 KiB
Python
# Copyright 2012 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import collections.abc
|
|
import json
|
|
import typing as ty
|
|
from urllib import parse
|
|
from urllib import request
|
|
|
|
from openstack import exceptions
|
|
from openstack.orchestration.util import environment_format
|
|
from openstack.orchestration.util import template_format
|
|
from openstack.orchestration.util import utils
|
|
|
|
|
|
def get_template_contents(
|
|
template_file=None,
|
|
template_url=None,
|
|
template_object=None,
|
|
object_request=None,
|
|
files=None,
|
|
existing=False,
|
|
):
|
|
is_object = False
|
|
tpl = None
|
|
|
|
# Transform a bare file path to a file:// URL.
|
|
if template_file:
|
|
template_url = utils.normalise_file_path_to_url(template_file)
|
|
|
|
if template_url:
|
|
tpl = request.urlopen(template_url).read()
|
|
|
|
elif template_object:
|
|
is_object = True
|
|
template_url = template_object
|
|
tpl = object_request and object_request('GET', template_object)
|
|
elif existing:
|
|
return {}, None
|
|
else:
|
|
raise exceptions.SDKException(
|
|
'Must provide one of template_file,'
|
|
' template_url or template_object'
|
|
)
|
|
|
|
if not tpl:
|
|
raise exceptions.SDKException(
|
|
'Could not fetch template from %s' % template_url
|
|
)
|
|
|
|
try:
|
|
if isinstance(tpl, bytes):
|
|
tpl = tpl.decode('utf-8')
|
|
template = template_format.parse(tpl)
|
|
except ValueError as e:
|
|
raise exceptions.SDKException(
|
|
'Error parsing template %(url)s %(error)s'
|
|
% {'url': template_url, 'error': e}
|
|
)
|
|
|
|
tmpl_base_url = utils.base_url_for_url(template_url)
|
|
if files is None:
|
|
files = {}
|
|
resolve_template_get_files(
|
|
template, files, tmpl_base_url, is_object, object_request
|
|
)
|
|
return files, template
|
|
|
|
|
|
def resolve_template_get_files(
|
|
template, files, template_base_url, is_object=False, object_request=None
|
|
):
|
|
def ignore_if(key, value):
|
|
if key != 'get_file' and key != 'type':
|
|
return True
|
|
if not isinstance(value, str):
|
|
return True
|
|
if key == 'type' and not value.endswith(('.yaml', '.template')):
|
|
return True
|
|
return False
|
|
|
|
def recurse_if(value):
|
|
return isinstance(value, (dict, list))
|
|
|
|
get_file_contents(
|
|
template,
|
|
files,
|
|
template_base_url,
|
|
ignore_if,
|
|
recurse_if,
|
|
is_object,
|
|
object_request,
|
|
)
|
|
|
|
|
|
def is_template(file_content):
|
|
try:
|
|
if isinstance(file_content, bytes):
|
|
file_content = file_content.decode('utf-8')
|
|
template_format.parse(file_content)
|
|
except (ValueError, TypeError):
|
|
return False
|
|
return True
|
|
|
|
|
|
def get_file_contents(
|
|
from_data,
|
|
files,
|
|
base_url=None,
|
|
ignore_if=None,
|
|
recurse_if=None,
|
|
is_object=False,
|
|
object_request=None,
|
|
):
|
|
if recurse_if and recurse_if(from_data):
|
|
if isinstance(from_data, dict):
|
|
recurse_data = from_data.values()
|
|
else:
|
|
recurse_data = from_data
|
|
for value in recurse_data:
|
|
get_file_contents(
|
|
value,
|
|
files,
|
|
base_url,
|
|
ignore_if,
|
|
recurse_if,
|
|
is_object,
|
|
object_request,
|
|
)
|
|
|
|
if isinstance(from_data, dict):
|
|
for key, value in from_data.items():
|
|
if ignore_if and ignore_if(key, value):
|
|
continue
|
|
|
|
if base_url and not base_url.endswith('/'):
|
|
base_url = base_url + '/'
|
|
|
|
str_url = parse.urljoin(base_url, value)
|
|
if str_url not in files:
|
|
if is_object and object_request:
|
|
file_content = object_request('GET', str_url)
|
|
else:
|
|
file_content = utils.read_url_content(str_url)
|
|
if is_template(file_content):
|
|
if is_object:
|
|
template = get_template_contents(
|
|
template_object=str_url,
|
|
files=files,
|
|
object_request=object_request,
|
|
)[1]
|
|
else:
|
|
template = get_template_contents(
|
|
template_url=str_url, files=files
|
|
)[1]
|
|
file_content = json.dumps(template)
|
|
files[str_url] = file_content
|
|
# replace the data value with the normalised absolute URL
|
|
from_data[key] = str_url
|
|
|
|
|
|
def deep_update(old, new):
|
|
'''Merge nested dictionaries.'''
|
|
|
|
# Prevents an error if in a previous iteration
|
|
# old[k] = None but v[k] = {...},
|
|
if old is None:
|
|
old = {}
|
|
|
|
for k, v in new.items():
|
|
if isinstance(v, collections.abc.Mapping):
|
|
r = deep_update(old.get(k, {}), v)
|
|
old[k] = r
|
|
else:
|
|
old[k] = new[k]
|
|
return old
|
|
|
|
|
|
def process_multiple_environments_and_files(
|
|
env_paths=None,
|
|
template=None,
|
|
template_url=None,
|
|
env_path_is_object=None,
|
|
object_request=None,
|
|
env_list_tracker=None,
|
|
):
|
|
"""Reads one or more environment files.
|
|
|
|
Reads in each specified environment file and returns a dictionary
|
|
of the filenames->contents (suitable for the files dict)
|
|
and the consolidated environment (after having applied the correct
|
|
overrides based on order).
|
|
|
|
If a list is provided in the env_list_tracker parameter, the behavior
|
|
is altered to take advantage of server-side environment resolution.
|
|
Specifically, this means:
|
|
|
|
* Populating env_list_tracker with an ordered list of environment file
|
|
URLs to be passed to the server
|
|
* Including the contents of each environment file in the returned
|
|
files dict, keyed by one of the URLs in env_list_tracker
|
|
|
|
:param env_paths: list of paths to the environment files to load; if
|
|
None, empty results will be returned
|
|
:type env_paths: list or None
|
|
:param template: unused; only included for API compatibility
|
|
:param template_url: unused; only included for API compatibility
|
|
:param env_list_tracker: if specified, environment filenames will be
|
|
stored within
|
|
:type env_list_tracker: list or None
|
|
:return: tuple of files dict and a dict of the consolidated environment
|
|
:rtype: tuple
|
|
"""
|
|
merged_files: ty.Dict[str, str] = {}
|
|
merged_env: ty.Dict[str, ty.Dict] = {}
|
|
|
|
# If we're keeping a list of environment files separately, include the
|
|
# contents of the files in the files dict
|
|
include_env_in_files = env_list_tracker is not None
|
|
|
|
if env_paths:
|
|
for env_path in env_paths:
|
|
files, env = process_environment_and_files(
|
|
env_path=env_path,
|
|
template=template,
|
|
template_url=template_url,
|
|
env_path_is_object=env_path_is_object,
|
|
object_request=object_request,
|
|
include_env_in_files=include_env_in_files,
|
|
)
|
|
|
|
# 'files' looks like {"filename1": contents, "filename2": contents}
|
|
# so a simple update is enough for merging
|
|
merged_files.update(files)
|
|
|
|
# 'env' can be a deeply nested dictionary, so a simple update is
|
|
# not enough
|
|
merged_env = deep_update(merged_env, env)
|
|
|
|
if env_list_tracker is not None:
|
|
env_url = utils.normalise_file_path_to_url(env_path)
|
|
env_list_tracker.append(env_url)
|
|
|
|
return merged_files, merged_env
|
|
|
|
|
|
def process_environment_and_files(
|
|
env_path=None,
|
|
template=None,
|
|
template_url=None,
|
|
env_path_is_object=None,
|
|
object_request=None,
|
|
include_env_in_files=False,
|
|
):
|
|
"""Loads a single environment file.
|
|
|
|
Returns an entry suitable for the files dict which maps the environment
|
|
filename to its contents.
|
|
|
|
:param env_path: full path to the file to load
|
|
:type env_path: str or None
|
|
:param include_env_in_files: if specified, the raw environment file itself
|
|
will be included in the returned files dict
|
|
:type include_env_in_files: bool
|
|
:return: tuple of files dict and the loaded environment as a dict
|
|
:rtype: (dict, dict)
|
|
"""
|
|
files: ty.Dict[str, str] = {}
|
|
env: ty.Dict[str, ty.Dict] = {}
|
|
|
|
is_object = env_path_is_object and env_path_is_object(env_path)
|
|
|
|
if is_object:
|
|
raw_env = object_request and object_request('GET', env_path)
|
|
env = environment_format.parse(raw_env)
|
|
env_base_url = utils.base_url_for_url(env_path)
|
|
|
|
resolve_environment_urls(
|
|
env.get('resource_registry'),
|
|
files,
|
|
env_base_url,
|
|
is_object=True,
|
|
object_request=object_request,
|
|
)
|
|
|
|
elif env_path:
|
|
env_url = utils.normalise_file_path_to_url(env_path)
|
|
env_base_url = utils.base_url_for_url(env_url)
|
|
raw_env = request.urlopen(env_url).read()
|
|
|
|
env = environment_format.parse(raw_env)
|
|
|
|
resolve_environment_urls(
|
|
env.get('resource_registry'), files, env_base_url
|
|
)
|
|
|
|
if include_env_in_files:
|
|
files[env_url] = json.dumps(env)
|
|
|
|
return files, env
|
|
|
|
|
|
def resolve_environment_urls(
|
|
resource_registry,
|
|
files,
|
|
env_base_url,
|
|
is_object=False,
|
|
object_request=None,
|
|
):
|
|
"""Handles any resource URLs specified in an environment.
|
|
|
|
:param resource_registry: mapping of type name to template filename
|
|
:type resource_registry: dict
|
|
:param files: dict to store loaded file contents into
|
|
:type files: dict
|
|
:param env_base_url: base URL to look in when loading files
|
|
:type env_base_url: str or None
|
|
"""
|
|
if resource_registry is None:
|
|
return
|
|
|
|
rr = resource_registry
|
|
base_url = rr.get('base_url', env_base_url)
|
|
|
|
def ignore_if(key, value):
|
|
if key == 'base_url':
|
|
return True
|
|
if isinstance(value, dict):
|
|
return True
|
|
if '::' in value:
|
|
# Built in providers like: "X::Compute::Server"
|
|
# don't need downloading.
|
|
return True
|
|
if key in ['hooks', 'restricted_actions']:
|
|
return True
|
|
|
|
get_file_contents(
|
|
rr,
|
|
files,
|
|
base_url,
|
|
ignore_if,
|
|
is_object=is_object,
|
|
object_request=object_request,
|
|
)
|
|
|
|
for res_name, res_dict in rr.get('resources', {}).items():
|
|
res_base_url = res_dict.get('base_url', base_url)
|
|
get_file_contents(
|
|
res_dict,
|
|
files,
|
|
res_base_url,
|
|
ignore_if,
|
|
is_object=is_object,
|
|
object_request=object_request,
|
|
)
|