openstack/tripleo-heat-templates
Code Issues Proposed changes

Merge "Make yaml-validate.py work with new t-h-t layout"

Browse Source
This commit is contained in:
Zuul 2020-02-21 20:58:38 +00:00 committed by Gerrit Code Review
commit 38a95f12b1
2 changed files with 92 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
deployment/gnocchi/gnocchi-api-container-puppet.yaml
View File

@ -117,6 +117,9 @@ resources:
ContainersCommon:
type: ../containers-common.yaml
MySQLClient:
type: ../../deployment/database/mysql-client.yaml
GnocchiServiceBase:
type: ./gnocchi-base.yaml
properties:
@ -226,8 +229,11 @@ outputs:
puppet_config:
config_volume: gnocchi
puppet_tags: gnocchi_api_paste_ini,gnocchi_config
step_config: |
include ::tripleo::profile::base::gnocchi::api
step_config:
list_join:
- "\n"
- - "include ::tripleo::profile::base::gnocchi::api"
- {get_attr: [MySQLClient, role_data, step_config]}
config_image: {get_param: ContainerGnocchiConfigImage}
kolla_config:
/var/lib/kolla/config_files/gnocchi_api.json:

96
tools/yaml-validate.py
View File

@ -14,12 +14,17 @@
import argparse
import os
import re
import six
import sys
import traceback
import yaml
from copy import copy
def is_string(value):
return isinstance(value, six.string_types)
# Only permit the template alias versions.
# The current template version should be the last element.
# As tripleo-heat-templates is a branched repository, this
@ -43,7 +48,10 @@ envs_containing_endpoint_map = ['no-tls-endpoints-public-ip.yaml',
'tls-endpoints-public-ip.yaml',
'tls-everywhere-endpoints-dns.yaml']
ENDPOINT_MAP_FILE = 'endpoint_map.yaml'
OPTIONAL_SECTIONS = ['cellv2_discovery']
OPTIONAL_SECTIONS = ['ansible_group_vars',
'cellv2_discovery',
'firewall_rules',
'keystone_resources']
REQUIRED_DOCKER_SECTIONS = ['service_name', 'docker_config', 'puppet_config',
'config_settings']
OPTIONAL_DOCKER_SECTIONS = ['container_puppet_tasks', 'upgrade_tasks',
@ -60,6 +68,38 @@ OPTIONAL_DOCKER_SECTIONS = ['container_puppet_tasks', 'upgrade_tasks',
'container_config_scripts', 'step_config',
'monitoring_subscription', 'scale_tasks',
'external_update_tasks', 'external_upgrade_tasks']
REQUIRED_DOCKER_SECTIONS_OVERRIDES = {
# Runs puppet within a container
'./deployment/neutron/neutron-agents-ib-config-container-puppet.yaml': [
'service_name',
'docker_config',
'config_settings'
],
# Just sets hieradata
'./deployment/neutron/neutron-ovn-dpdk-config-container-puppet.yaml': [
'service_name',
'config_settings'
],
# Does not deploy container
'./deployment/ceilometer/ceilometer-base-container-puppet.yaml': [
'service_name',
'config_settings'
],
# Does not manage container using docker_config
'./deployment/nova/nova-libvirt-guests-container-puppet.yaml': [
'service_name',
'puppet_config',
'config_settings'
],
# Inherits sections
'./deployment/haproxy/haproxy-edge-container-puppet.yaml': [
'service_name',
'config_settings'
],
'./deployment/glance/glance-api-edge-container-puppet.yaml': [
'service_name',
],
}
# ansible tasks cannot be an empty dict or ansible is unhappy
ANSIBLE_TASKS_SECTIONS = ['upgrade_tasks', 'pre_upgrade_rolling_tasks',
'fast_forward_upgrade_tasks',
@ -220,6 +260,8 @@ VALIDATE_DOCKER_OVERRIDE = {
# deployment/rabbitmq/rabbitmq-messaging-notify-shared-puppet.yaml does not
# deploy container
'./deployment/rabbitmq/rabbitmq-messaging-notify-shared-puppet.yaml': False,
# Does not follow the filename pattern
'./deployment/multipathd/multipathd-container.yaml': True
}
DEPLOYMENT_RESOURCE_TYPES = [
'OS::Heat::SoftwareDeploymentGroup',
@ -257,6 +299,14 @@ HEAT_OUTPUTS_EXCLUSIONS = [
'./extraconfig/pre_network/host_config_and_reboot.yaml'
]
CONFIG_IMAGE_OVERRIDES = {
'ContainerSwiftRingbuilderConfigImage': 'ContainerSwiftConfigImage'
}
SERVICE_NAME_OVERRIDE = {
'./deployment/rabbitmq/rabbitmq-messaging-pacemaker-puppet.yaml': 'rabbitmq',
}
def exit_usage():
print('Usage %s <yaml file or directory>' % sys.argv[0])
@ -612,9 +662,12 @@ def validate_docker_service_mysql_usage(filename, tpl):
continue
newfilename = \
os.path.normpath(os.path.join(os.path.dirname(incfile), f))
if not os.path.exists(newfilename) and \
os.path.exists(newfilename.replace('.yaml', '.j2.yaml')):
return # Skip for now if it's templated
with open(newfilename, 'r') as newfile:
newtmp = yaml.load(newfile.read(), Loader=yaml.SafeLoader)
read_all(newfile, newtmp)
read_all(newfilename, newtmp)
read_all(filename, tpl)
if search(all_content, match_use_mysql_protocol, no_op):
@ -650,15 +703,24 @@ def validate_docker_service(filename, tpl):
% filename)
return 1
role_data = tpl['outputs']['role_data']['value']
if list(role_data.keys()) == ['map_merge']:
merged_role_data = {}
for part in role_data['map_merge']:
if list(part.keys()) == ['get_attr']:
continue
merged_role_data.update(part)
role_data = merged_role_data
for section_name in REQUIRED_DOCKER_SECTIONS:
for section_name in REQUIRED_DOCKER_SECTIONS_OVERRIDES.get(filename, REQUIRED_DOCKER_SECTIONS):
if section_name not in role_data:
# add an exception if both step_config is used in docker
# service, deployment/ceph-ansible/ceph-nfs.yaml uses
# additional step_config to add pacemaker resources
if (section_name == 'docker_config' and
role_data.get('step_config', '')):
continue
print('ERROR: %s appears to be a barematal-puppet service'
% (filename))
return 1
print('ERROR: %s is required in role_data for %s.'
% (section_name, filename))
return 1
@ -709,6 +771,10 @@ def validate_docker_service(filename, tpl):
config_volume = puppet_config.get('config_volume')
expected_config_image_parameter = \
"Container%sConfigImage" % to_camel_case(config_volume)
expected_config_image_parameter = CONFIG_IMAGE_OVERRIDES.get(
expected_config_image_parameter,
expected_config_image_parameter
)
if config_volume and expected_config_image_parameter not in tpl.get('parameters', []):
print('ERROR: Missing %s heat parameter for %s config_volume.'
% (expected_config_image_parameter, config_volume))
@ -784,14 +850,16 @@ def validate_service(filename, tpl):
print('ERROR: service_name is required in role_data for %s.'
% filename)
return 1
# service_name must match the beginning of the file name, but with an
# underscore
# service_name must match the beginning of the file name, but with an underscore
service_name = \
os.path.basename(filename).split('.')[0].replace("-", "_")
os.path.basename(filename).split('.')[0].rsplit('-', 2)[0].replace('-', '_')
if is_string(role_data['service_name']):
service_name = SERVICE_NAME_OVERRIDE.get(filename, service_name)
if not role_data['service_name'].startswith(service_name):
print('ERROR: service_name should match the beginning of the '
'filename: %s.'
% filename)
print('ERROR: service_name "%s" should match the beginning of the '
'filename: %s (%s).'
% (role_data['service_name'], os.path.basename(filename), service_name))
return 1
# if service connects to mysql, the uri should use option
# bind_address to avoid issues with VIP failover
@ -1099,18 +1167,18 @@ def validate(filename, param_map):
)
if VALIDATE_PUPPET_OVERRIDE.get(filename, False) or (
filename.startswith('./puppet/services/') and
re.search(r'^\.\/deployment\/.+-(baremetal|pacemaker)-puppet.yaml$', filename) and
VALIDATE_PUPPET_OVERRIDE.get(filename, True)):
retval |= validate_service(filename, tpl)
if re.search(r'(puppet|docker)\/services', filename) or \
re.search(r'deployment\/', filename):
re.search(r'^\.\/deployment\/', filename):
retval |= validate_service_hiera_interpol(filename, tpl)
if filename.startswith('./docker/services/logging/'):
if re.search(r'^\.\/deployment\/logging\/(files|stdout)\/', filename):
retval |= validate_docker_logging_template(filename, tpl)
elif VALIDATE_DOCKER_OVERRIDE.get(filename, False) or (
filename.startswith('./docker/services/') and
re.search(r'^\.\/deployment\/.+-container-puppet.yaml$', filename) and
VALIDATE_DOCKER_OVERRIDE.get(filename, True)):
retval |= validate_docker_service(filename, tpl)