kolla-ansible/tools/validate-all-file.py

202 lines
7.1 KiB
Python
Raw Normal View History

#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import fnmatch
import json
import logging
import os
import re
import sys
import jinja2
import yaml
Implement IPv6 support in the control plane Introduce kolla_address filter. Introduce put_address_in_context filter. Add AF config to vars. Address contexts: - raw (default): <ADDR> - memcache: inet6:[<ADDR>] - url: [<ADDR>] Other changes: globals.yml - mention just IP in comment prechecks/port_checks (api_intf) - kolla_address handles validation 3x interface conditional (swift configs: replication/storage) 2x interface variable definition with hostname (haproxy listens; api intf) 1x interface variable definition with hostname with bifrost exclusion (baremetal pre-install /etc/hosts; api intf) neutron's ml2 'overlay_ip_version' set to 6 for IPv6 on tunnel network basic multinode source CI job for IPv6 prechecks for rabbitmq and qdrouterd use proper NSS database now MariaDB Galera Cluster WSREP SST mariabackup workaround (socat and IPv6) Ceph naming workaround in CI TODO: probably needs documenting RabbitMQ IPv6-only proto_dist Ceph ms switch to IPv6 mode Remove neutron-server ml2_type_vxlan/vxlan_group setting as it is not used (let's avoid any confusion) and could break setups without proper multicast routing if it started working (also IPv4-only) haproxy upgrade checks for slaves based on ipv6 addresses TODO: ovs-dpdk grabs ipv4 network address (w/ prefix len / submask) not supported, invalid by default because neutron_external has no address No idea whether ovs-dpdk works at all atm. ml2 for xenapi Xen is not supported too well. This would require working with XenAPI facts. rp_filter setting This would require meddling with ip6tables (there is no sysctl param). By default nothing is dropped. Unlikely we really need it. ironic dnsmasq is configured IPv4-only dnsmasq needs DHCPv6 options and testing in vivo. KNOWN ISSUES (beyond us): One cannot use IPv6 address to reference the image for docker like we currently do, see: https://github.com/moby/moby/issues/39033 (docker_registry; docker API 400 - invalid reference format) workaround: use hostname/FQDN RabbitMQ may fail to bind to IPv6 if hostname resolves also to IPv4. This is due to old RabbitMQ versions available in images. IPv4 is preferred by default and may fail in the IPv6-only scenario. This should be no problem in real life as IPv6-only is indeed IPv6-only. Also, when new RabbitMQ (3.7.16/3.8+) makes it into images, this will no longer be relevant as we supply all the necessary config. See: https://github.com/rabbitmq/rabbitmq-server/pull/1982 For reliable runs, at least Ansible 2.8 is required (2.8.5 confirmed to work well). Older Ansible versions are known to miss IPv6 addresses in interface facts. This may affect redeploys, reconfigures and upgrades which run after VIP address is assigned. See: https://github.com/ansible/ansible/issues/63227 Bifrost Train does not support IPv6 deployments. See: https://storyboard.openstack.org/#!/story/2006689 Change-Id: Ia34e6916ea4f99e9522cd2ddde03a0a4776f7e2c Implements: blueprint ipv6-control-plane Signed-off-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>
2019-09-11 20:47:00 +02:00
from kolla_ansible.put_address_in_context import put_address_in_context
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
NEWLINE_EOF_INCLUDE_PATTERNS = ['*.j2', '*.yml', '*.py', '*.sh']
NEWLINE_EOF_EXCLUDE_PATTERNS = ['.tox', '.testrepository', '.git']
# Render json file by using jinja2 template is OK
JSON_J2_INCLUDE_PATTERNS = ['*.json.j2', '*.json']
JSON_J2_EXCLUDE_PATTERNS = ['.tox', '.testrepository', '.git']
YAML_INCLUDE_PATTERNS = ['*.yml']
YAML_EXCLUDE_PATTERNS = ['.tox', '.testrepository', '.git',
'defaults', 'templates', 'vars']
Implement IPv6 support in the control plane Introduce kolla_address filter. Introduce put_address_in_context filter. Add AF config to vars. Address contexts: - raw (default): <ADDR> - memcache: inet6:[<ADDR>] - url: [<ADDR>] Other changes: globals.yml - mention just IP in comment prechecks/port_checks (api_intf) - kolla_address handles validation 3x interface conditional (swift configs: replication/storage) 2x interface variable definition with hostname (haproxy listens; api intf) 1x interface variable definition with hostname with bifrost exclusion (baremetal pre-install /etc/hosts; api intf) neutron's ml2 'overlay_ip_version' set to 6 for IPv6 on tunnel network basic multinode source CI job for IPv6 prechecks for rabbitmq and qdrouterd use proper NSS database now MariaDB Galera Cluster WSREP SST mariabackup workaround (socat and IPv6) Ceph naming workaround in CI TODO: probably needs documenting RabbitMQ IPv6-only proto_dist Ceph ms switch to IPv6 mode Remove neutron-server ml2_type_vxlan/vxlan_group setting as it is not used (let's avoid any confusion) and could break setups without proper multicast routing if it started working (also IPv4-only) haproxy upgrade checks for slaves based on ipv6 addresses TODO: ovs-dpdk grabs ipv4 network address (w/ prefix len / submask) not supported, invalid by default because neutron_external has no address No idea whether ovs-dpdk works at all atm. ml2 for xenapi Xen is not supported too well. This would require working with XenAPI facts. rp_filter setting This would require meddling with ip6tables (there is no sysctl param). By default nothing is dropped. Unlikely we really need it. ironic dnsmasq is configured IPv4-only dnsmasq needs DHCPv6 options and testing in vivo. KNOWN ISSUES (beyond us): One cannot use IPv6 address to reference the image for docker like we currently do, see: https://github.com/moby/moby/issues/39033 (docker_registry; docker API 400 - invalid reference format) workaround: use hostname/FQDN RabbitMQ may fail to bind to IPv6 if hostname resolves also to IPv4. This is due to old RabbitMQ versions available in images. IPv4 is preferred by default and may fail in the IPv6-only scenario. This should be no problem in real life as IPv6-only is indeed IPv6-only. Also, when new RabbitMQ (3.7.16/3.8+) makes it into images, this will no longer be relevant as we supply all the necessary config. See: https://github.com/rabbitmq/rabbitmq-server/pull/1982 For reliable runs, at least Ansible 2.8 is required (2.8.5 confirmed to work well). Older Ansible versions are known to miss IPv6 addresses in interface facts. This may affect redeploys, reconfigures and upgrades which run after VIP address is assigned. See: https://github.com/ansible/ansible/issues/63227 Bifrost Train does not support IPv6 deployments. See: https://storyboard.openstack.org/#!/story/2006689 Change-Id: Ia34e6916ea4f99e9522cd2ddde03a0a4776f7e2c Implements: blueprint ipv6-control-plane Signed-off-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>
2019-09-11 20:47:00 +02:00
KOLLA_NETWORKS = [
'api',
'storage',
'cluster',
'swift_storage',
'swift_replication',
'migration',
'tunnel',
'octavia_network',
'bifrost_network',
'dns', # designate
]
logging.basicConfig()
LOG = logging.getLogger(__name__)
def check_newline_eof():
includes = r'|'.join([fnmatch.translate(x)
for x in NEWLINE_EOF_INCLUDE_PATTERNS])
excludes = r'|'.join([fnmatch.translate(x)
for x in NEWLINE_EOF_EXCLUDE_PATTERNS])
return_code = 0
def has_newline_eof(path):
with open(path, 'r') as f:
data = f.read()
if data and data[-1] != '\n':
LOG.error('%s file error: no newline at end of file', path)
return False
return True
for root, dirs, files in os.walk(PROJECT_ROOT):
dirs[:] = [d for d in dirs if not re.match(excludes, d)]
for f in files:
if not re.match(excludes, f) and re.match(includes, f):
if not has_newline_eof(os.path.join(root, f)):
return_code = 1
return return_code
def check_json_j2():
includes = r'|'.join([fnmatch.translate(x)
for x in JSON_J2_INCLUDE_PATTERNS])
excludes = r'|'.join([fnmatch.translate(x)
for x in JSON_J2_EXCLUDE_PATTERNS])
return_code = 0
def bool_filter(value):
return True
def basename_filter(text):
return text.split('\\')[-1]
Implement IPv6 support in the control plane Introduce kolla_address filter. Introduce put_address_in_context filter. Add AF config to vars. Address contexts: - raw (default): <ADDR> - memcache: inet6:[<ADDR>] - url: [<ADDR>] Other changes: globals.yml - mention just IP in comment prechecks/port_checks (api_intf) - kolla_address handles validation 3x interface conditional (swift configs: replication/storage) 2x interface variable definition with hostname (haproxy listens; api intf) 1x interface variable definition with hostname with bifrost exclusion (baremetal pre-install /etc/hosts; api intf) neutron's ml2 'overlay_ip_version' set to 6 for IPv6 on tunnel network basic multinode source CI job for IPv6 prechecks for rabbitmq and qdrouterd use proper NSS database now MariaDB Galera Cluster WSREP SST mariabackup workaround (socat and IPv6) Ceph naming workaround in CI TODO: probably needs documenting RabbitMQ IPv6-only proto_dist Ceph ms switch to IPv6 mode Remove neutron-server ml2_type_vxlan/vxlan_group setting as it is not used (let's avoid any confusion) and could break setups without proper multicast routing if it started working (also IPv4-only) haproxy upgrade checks for slaves based on ipv6 addresses TODO: ovs-dpdk grabs ipv4 network address (w/ prefix len / submask) not supported, invalid by default because neutron_external has no address No idea whether ovs-dpdk works at all atm. ml2 for xenapi Xen is not supported too well. This would require working with XenAPI facts. rp_filter setting This would require meddling with ip6tables (there is no sysctl param). By default nothing is dropped. Unlikely we really need it. ironic dnsmasq is configured IPv4-only dnsmasq needs DHCPv6 options and testing in vivo. KNOWN ISSUES (beyond us): One cannot use IPv6 address to reference the image for docker like we currently do, see: https://github.com/moby/moby/issues/39033 (docker_registry; docker API 400 - invalid reference format) workaround: use hostname/FQDN RabbitMQ may fail to bind to IPv6 if hostname resolves also to IPv4. This is due to old RabbitMQ versions available in images. IPv4 is preferred by default and may fail in the IPv6-only scenario. This should be no problem in real life as IPv6-only is indeed IPv6-only. Also, when new RabbitMQ (3.7.16/3.8+) makes it into images, this will no longer be relevant as we supply all the necessary config. See: https://github.com/rabbitmq/rabbitmq-server/pull/1982 For reliable runs, at least Ansible 2.8 is required (2.8.5 confirmed to work well). Older Ansible versions are known to miss IPv6 addresses in interface facts. This may affect redeploys, reconfigures and upgrades which run after VIP address is assigned. See: https://github.com/ansible/ansible/issues/63227 Bifrost Train does not support IPv6 deployments. See: https://storyboard.openstack.org/#!/story/2006689 Change-Id: Ia34e6916ea4f99e9522cd2ddde03a0a4776f7e2c Implements: blueprint ipv6-control-plane Signed-off-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>
2019-09-11 20:47:00 +02:00
def kolla_address_filter_mock(network_name, hostname=None):
# no validation is possible for the hostname
if network_name not in KOLLA_NETWORKS:
raise ValueError("{network_name} not in KOLLA_NETWORKS"
.format(network_name=network_name))
return "127.0.0.1"
# Mock ansible hostvars variable, which is a nested dict
def hostvars():
return collections.defaultdict(hostvars)
# Mock Ansible groups variable, which is a dict of lists.
def groups():
return collections.defaultdict(list)
def validate_json_j2(root, filename):
env = jinja2.Environment( # nosec: not used to render HTML
loader=jinja2.FileSystemLoader(root))
env.filters['bool'] = bool_filter
env.filters['basename'] = basename_filter
Implement IPv6 support in the control plane Introduce kolla_address filter. Introduce put_address_in_context filter. Add AF config to vars. Address contexts: - raw (default): <ADDR> - memcache: inet6:[<ADDR>] - url: [<ADDR>] Other changes: globals.yml - mention just IP in comment prechecks/port_checks (api_intf) - kolla_address handles validation 3x interface conditional (swift configs: replication/storage) 2x interface variable definition with hostname (haproxy listens; api intf) 1x interface variable definition with hostname with bifrost exclusion (baremetal pre-install /etc/hosts; api intf) neutron's ml2 'overlay_ip_version' set to 6 for IPv6 on tunnel network basic multinode source CI job for IPv6 prechecks for rabbitmq and qdrouterd use proper NSS database now MariaDB Galera Cluster WSREP SST mariabackup workaround (socat and IPv6) Ceph naming workaround in CI TODO: probably needs documenting RabbitMQ IPv6-only proto_dist Ceph ms switch to IPv6 mode Remove neutron-server ml2_type_vxlan/vxlan_group setting as it is not used (let's avoid any confusion) and could break setups without proper multicast routing if it started working (also IPv4-only) haproxy upgrade checks for slaves based on ipv6 addresses TODO: ovs-dpdk grabs ipv4 network address (w/ prefix len / submask) not supported, invalid by default because neutron_external has no address No idea whether ovs-dpdk works at all atm. ml2 for xenapi Xen is not supported too well. This would require working with XenAPI facts. rp_filter setting This would require meddling with ip6tables (there is no sysctl param). By default nothing is dropped. Unlikely we really need it. ironic dnsmasq is configured IPv4-only dnsmasq needs DHCPv6 options and testing in vivo. KNOWN ISSUES (beyond us): One cannot use IPv6 address to reference the image for docker like we currently do, see: https://github.com/moby/moby/issues/39033 (docker_registry; docker API 400 - invalid reference format) workaround: use hostname/FQDN RabbitMQ may fail to bind to IPv6 if hostname resolves also to IPv4. This is due to old RabbitMQ versions available in images. IPv4 is preferred by default and may fail in the IPv6-only scenario. This should be no problem in real life as IPv6-only is indeed IPv6-only. Also, when new RabbitMQ (3.7.16/3.8+) makes it into images, this will no longer be relevant as we supply all the necessary config. See: https://github.com/rabbitmq/rabbitmq-server/pull/1982 For reliable runs, at least Ansible 2.8 is required (2.8.5 confirmed to work well). Older Ansible versions are known to miss IPv6 addresses in interface facts. This may affect redeploys, reconfigures and upgrades which run after VIP address is assigned. See: https://github.com/ansible/ansible/issues/63227 Bifrost Train does not support IPv6 deployments. See: https://storyboard.openstack.org/#!/story/2006689 Change-Id: Ia34e6916ea4f99e9522cd2ddde03a0a4776f7e2c Implements: blueprint ipv6-control-plane Signed-off-by: Radosław Piliszek <radoslaw.piliszek@gmail.com>
2019-09-11 20:47:00 +02:00
env.filters['kolla_address'] = kolla_address_filter_mock
env.filters['put_address_in_context'] = \
put_address_in_context
template = env.get_template(filename)
# Mock ansible variables.
context = {
'hostvars': hostvars(),
'groups': groups(),
'cluster_interface': 'cluster_interface',
'storage_interface': 'storage_interface',
'inventory_hostname': 'hostname'
}
data = template.render(**context)
json.loads(data)
for root, dirs, files in os.walk(PROJECT_ROOT):
dirs[:] = [d for d in dirs if not re.match(excludes, d)]
for filename in files:
if not re.match(excludes, filename) and \
re.match(includes, filename):
fullpath = os.path.join(root, filename)
try:
validate_json_j2(root, filename)
except (ValueError, jinja2.exceptions.TemplateError):
return_code = 1
LOG.exception('%s file error', fullpath)
return return_code
def check_docker_become():
"""All tasks that use Docker should have 'become: true'."""
includes = r'|'.join([fnmatch.translate(x)
for x in YAML_INCLUDE_PATTERNS])
excludes = r'|'.join([fnmatch.translate(x)
for x in YAML_EXCLUDE_PATTERNS])
docker_modules = ('kolla_docker', 'kolla_container_facts', 'kolla_toolbox')
cmd_modules = ('command', 'shell')
return_code = 0
roles_path = os.path.join(PROJECT_ROOT, 'ansible', 'roles')
for root, dirs, files in os.walk(roles_path):
dirs[:] = [d for d in dirs if not re.match(excludes, d)]
for filename in files:
if not re.match(excludes, filename) and \
re.match(includes, filename):
fullpath = os.path.join(root, filename)
with open(fullpath) as fp:
tasks = yaml.safe_load(fp)
tasks = tasks or []
for task in tasks:
for module in docker_modules:
if module in task and not task.get('become'):
return_code = 1
LOG.error("Use of %s module without become in "
"task %s in %s",
module, task['name'], fullpath)
for module in cmd_modules:
docker_without_become = False
if (module in task and not task.get('become')):
if (isinstance(task[module], str) and
(task[module]).startswith('docker')):
docker_without_become = True
if (isinstance(task[module], dict) and
task[module]['cmd'].startswith('docker')):
docker_without_become = True
if docker_without_become:
return_code = 1
LOG.error("Use of docker in %s module without "
"become in task %s in %s",
module, task['name'], fullpath)
return return_code
def main():
checks = (
check_newline_eof,
check_json_j2,
check_docker_become,
)
return sum([check() for check in checks])
if __name__ == "__main__":
sys.exit(main())