Fix trailing slashes in bind-mounts and linting

Podman is not always happy with bind-mounts ending
with / (See I094120f7f2f6bfcfc0cc5843aa1b23629cd90a23)

Follow up If951f9643d67574c1225301aab7c9e4b0d316b7f
with that YAML linter couldn't catch.

Improve linter to process all volumes in templates, including
common/logging services templates, and neither puppet, not ansible
ones, like deployment/deprecated/multipathd-container.yaml.

Change-Id: Ia517b34c9d633101502bd8788e7b8764e75bbe64
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>
This commit is contained in:
Bogdan Dobrelya 2022-03-03 17:32:58 +01:00 committed by Cedric Jeanneret
parent ab71ee8d79
commit d0ca9fe631
19 changed files with 105 additions and 73 deletions

View File

@ -358,7 +358,7 @@ outputs:
- - 'ro'
- /lib/modules:/lib/modules:ro
- /dev:/dev
- /run/:/run/
- /run:/run
- /sys:/sys
- /var/lib/cinder:/var/lib/cinder:z
- /var/lib/iscsi:/var/lib/iscsi:z
@ -404,7 +404,7 @@ outputs:
- - '/var/lib/kolla/config_files/src-ceph'
- - 'ro'
- /dev:/dev
- /run/:/run/
- /run:/run
- /sys:/sys
- /lib/modules:/lib/modules:ro
- /var/lib/cinder:/var/lib/cinder:z

View File

@ -141,7 +141,7 @@ outputs:
- /var/lib/config-data/puppet-generated/multipathd:/var/lib/kolla/config_files/src:ro
- /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro
- /dev:/dev
- /run/:/run/
- /run:/run
- /sys:/sys
- /lib/modules:/lib/modules:ro
- /var/lib/iscsi:/var/lib/iscsi:z

View File

@ -211,7 +211,7 @@ outputs:
config_image: *etcd_config_image
# TODO(mwhahaha): need to check if this works correctly or we need to switch to a bootstrap execution
volumes:
- /var/lib/config-data/puppet-generated/etcd/etc/etcd/:/etc/etcd:ro
- /var/lib/config-data/puppet-generated/etcd/etc/etcd:/etc/etcd:ro
- /var/lib/etcd:/var/lib/etcd:ro
deploy_steps_tasks:
if:

View File

@ -51,7 +51,7 @@ outputs:
test: '[ -p /var/log/httpd/error_log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/error_log && mkfifo /var/log/httpd/error_log && while true; do cat /var/log/httpd/error_log; done']
volumes:
- BarbicanApiLogs:/var/log/
- BarbicanApiLogs:/var/log
barbican_api_apache_access_logs:
start_order: 1
image: {get_attr: [RoleParametersValue, value, ContainerBarbicanApiImage]}
@ -62,7 +62,7 @@ outputs:
test: '[ -p /var/log/httpd/access.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/access.log && mkfifo /var/log/httpd/access.log && while true; do cat /var/log/httpd/access.log; done']
volumes:
- BarbicanApiLogs:/var/log/
- BarbicanApiLogs:/var/log
barbican_api_logs:
start_order: 2
image: {get_attr: [RoleParametersValue, value, ContainerBarbicanApiImage]}
@ -73,11 +73,11 @@ outputs:
test: '[ -p /var/log/barbican/main.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/barbican && rm -f /var/log/barbican/main.log && mkfifo /var/log/barbican/main.log && while true; do cat /var/log/barbican/main.log; done']
volumes:
- BarbicanApiLogs:/var/log/
- BarbicanApiLogs:/var/log
volumes:
description: The volumes needed to log to stdout or a sidecar container.
value:
- BarbicanApiLogs:/var/log/
- BarbicanApiLogs:/var/log
host_prep_tasks:
description: Extra ansible tasks needed for logging to files in the host.
value: null

View File

@ -45,13 +45,13 @@ outputs:
volumes:
- /var/lib/kolla/config_files/rsyslog_sidecar.json:/var/lib/kolla/config_files/config.json:ro
- /var/lib/config-data/puppet-generated/rsyslog_sidecar:/var/lib/kolla/config_files/src:ro
- HAProxyLogs:/sockets/
- HAProxyLogs:/sockets
environment:
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
volumes:
description: The volumes needed to log to stdout or a sidecar container.
value:
- HAProxyLogs:/sockets/
- HAProxyLogs:/sockets
host_prep_tasks:
description: Extra ansible tasks needed for logging to files in the host.
value: null

View File

@ -48,7 +48,7 @@ outputs:
test: '[ -p /var/log/httpd/error_log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/error_log && mkfifo /var/log/httpd/error_log && while true; do cat /var/log/httpd/error_log; done']
volumes:
- HeatApiCfnLogs:/var/log/
- HeatApiCfnLogs:/var/log
heat_api_cfn_apache_access_logs:
start_order: 1
image: {get_attr: [RoleParametersValue, value, ContainerHeatApiCfnImage]}
@ -59,7 +59,7 @@ outputs:
test: '[ -p /var/log/httpd/access.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/access.log && mkfifo /var/log/httpd/access.log && while true; do cat /var/log/httpd/access.log; done']
volumes:
- HeatApiCfnLogs:/var/log/
- HeatApiCfnLogs:/var/log
heat_api_cfn_logs:
start_order: 2
image: {get_attr: [RoleParametersValue, value, ContainerHeatApiCfnImage]}
@ -70,11 +70,11 @@ outputs:
test: '[ -p /var/log/heat/heat_api_cfn.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/heat && rm -f /var/log/heat/heat_api_cfn.log && mkfifo /var/log/heat/heat_api_cfn.log && while true; do cat /var/log/heat/heat_api_cfn.log; done']
volumes:
- HeatApiCfnLogs:/var/log/
- HeatApiCfnLogs:/var/log
volumes:
description: The volumes needed to log to stdout or a sidecar container.
value:
- HeatApiCfnLogs:/var/log/
- HeatApiCfnLogs:/var/log
host_prep_tasks:
description: Extra ansible tasks needed for logging to files in the host.
value: null

View File

@ -48,7 +48,7 @@ outputs:
test: '[ -p /var/log/httpd/error_log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/error_log && mkfifo /var/log/httpd/error_log && while true; do cat /var/log/httpd/error_log; done']
volumes:
- HeatApiLogs:/var/log/
- HeatApiLogs:/var/log
heat_api_apache_access_logs:
start_order: 1
image: {get_attr: [RoleParametersValue, value, ContainerHeatApiImage]}
@ -59,7 +59,7 @@ outputs:
test: '[ -p /var/log/httpd/access.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/access.log && mkfifo /var/log/httpd/access.log && while true; do cat /var/log/httpd/access.log; done']
volumes:
- HeatApiLogs:/var/log/
- HeatApiLogs:/var/log
heat_api_logs:
start_order: 2
image: {get_attr: [RoleParametersValue, value, ContainerHeatApiImage]}
@ -70,11 +70,11 @@ outputs:
test: '[ -p /var/log/heat/heat_api.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/heat && rm -f /var/log/heat/heat_api.log && mkfifo /var/log/heat/heat_api.log && while true; do cat /var/log/heat/heat_api.log; done']
volumes:
- HeatApiLogs:/var/log/
- HeatApiLogs:/var/log
volumes:
description: The volumes needed to log to stdout or a sidecar container.
value:
- HeatApiLogs:/var/log/
- HeatApiLogs:/var/log
host_prep_tasks:
description: Extra ansible tasks needed for logging to files in the host.
value: null

View File

@ -48,7 +48,7 @@ outputs:
test: '[ -p /var/log/httpd/error_log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/error_log && mkfifo /var/log/httpd/error_log && while true; do cat /var/log/httpd/error_log; done']
volumes:
- KeystoneLogs:/var/log/
- KeystoneLogs:/var/log
keystone_apache_access_logs:
start_order: 1
image: {get_attr: [RoleParametersValue, value, ContainerKeystoneImage]}
@ -59,7 +59,7 @@ outputs:
test: '[ -p /var/log/httpd/access.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/access.log && mkfifo /var/log/httpd/access.log && while true; do cat /var/log/httpd/access.log; done']
volumes:
- KeystoneLogs:/var/log/
- KeystoneLogs:/var/log
keystone_logs:
start_order: 2
image: {get_attr: [RoleParametersValue, value, ContainerKeystoneImage]}
@ -70,7 +70,7 @@ outputs:
test: '[ -p /var/log/keystone/keystone.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/keystone && rm -f /var/log/keystone/keystone.log && mkfifo /var/log/keystone/keystone.log && while true; do cat /var/log/keystone/keystone.log; done']
volumes:
- KeystoneLogs:/var/log/
- KeystoneLogs:/var/log
environment:
description: Extra environment variables needed for the service to log to stdout.
value:
@ -78,7 +78,7 @@ outputs:
volumes:
description: The volumes needed to log to stdout or a sidecar container.
value:
- KeystoneLogs:/var/log/
- KeystoneLogs:/var/log
host_prep_tasks:
description: Extra ansible tasks needed for logging to files in the host.
value: null

View File

@ -37,7 +37,7 @@ outputs:
volumes:
description: The volumes needed to log to stdout or a sidecar container.
value:
- NovaApiLogs:/var/log/
- NovaApiLogs:/var/log
docker_config:
description: Extra containers needed for logging to stdout or a sidecar container.
value:
@ -52,7 +52,7 @@ outputs:
test: '[ -p /var/log/httpd/error_log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/error_log && mkfifo /var/log/httpd/error_log && while true; do cat /var/log/httpd/error_log; done']
volumes:
- NovaApiLogs:/var/log/
- NovaApiLogs:/var/log
nova_api_apache_access_logs:
start_order: 1
image: {get_attr: [RoleParametersValue, value, ContainerNovaApiImage]}
@ -63,7 +63,7 @@ outputs:
test: '[ -p /var/log/httpd/access.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/access.log && mkfifo /var/log/httpd/access.log && while true; do cat /var/log/httpd/access.log; done']
volumes:
- NovaApiLogs:/var/log/
- NovaApiLogs:/var/log
nova_api_logs:
start_order: 2
image: {get_attr: [RoleParametersValue, value, ContainerNovaApiImage]}
@ -74,7 +74,7 @@ outputs:
test: '[ -p /var/log/nova/nova-api.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/nova && rm -f /var/log/nova/nova-api.log && mkfifo /var/log/nova/nova-api.log && while true; do cat /var/log/nova/nova-api.log; done']
volumes:
- NovaApiLogs:/var/log/
- NovaApiLogs:/var/log
host_prep_tasks:
description: Extra ansible tasks needed for logging to files in the host.
value: null

View File

@ -37,7 +37,7 @@ outputs:
volumes:
description: The volumes needed to log to stdout or a sidecar container.
value:
- NovaMetadataLogs:/var/log/
- NovaMetadataLogs:/var/log
docker_config:
description: Extra containers needed for logging to stdout or a sidecar container.
value:
@ -52,7 +52,7 @@ outputs:
test: '[ -p /var/log/httpd/error_log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/error_log && mkfifo /var/log/httpd/error_log && while true; do cat /var/log/httpd/error_log; done']
volumes:
- NovaMetadataLogs:/var/log/
- NovaMetadataLogs:/var/log
nova_metadata_apache_access_logs:
start_order: 1
image: {get_attr: [RoleParametersValue, value, ContainerNovaMetadataImage]}
@ -63,7 +63,7 @@ outputs:
test: '[ -p /var/log/httpd/access.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/access.log && mkfifo /var/log/httpd/access.log && while true; do cat /var/log/httpd/access.log; done']
volumes:
- NovaMetadataLogs:/var/log/
- NovaMetadataLogs:/var/log
nova_metadata_logs:
start_order: 2
image: {get_attr: [RoleParametersValue, value, ContainerNovaMetadataImage]}
@ -74,7 +74,7 @@ outputs:
test: '[ -p /var/log/nova/nova-metadata-api.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/nova && rm -f /var/log/nova/nova-metadata-api.log && mkfifo /var/log/nova/nova-metadata-api.log && while true; do cat /var/log/nova/nova-metadata-api.log; done']
volumes:
- NovaMetadataLogs:/var/log/
- NovaMetadataLogs:/var/log
host_prep_tasks:
description: Extra ansible tasks needed for logging to files in the host.
value: null

View File

@ -37,7 +37,7 @@ outputs:
volumes:
description: The volumes needed to log to stdout or a sidecar container.
value:
- PlacementLogs:/var/log/
- PlacementLogs:/var/log
docker_config:
description: Extra containers needed for logging to stdout or a sidecar container.
value:
@ -52,7 +52,7 @@ outputs:
test: '[ -p /var/log/httpd/error_log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/error_log && mkfifo /var/log/httpd/error_log && while true; do cat /var/log/httpd/error_log; done']
volumes:
- PlacementLogs:/var/log/
- PlacementLogs:/var/log
placement_apache_access_logs:
start_order: 1
image: {get_attr: [RoleParametersValue, value, ContainerPlacementImage]}
@ -63,7 +63,7 @@ outputs:
test: '[ -p /var/log/httpd/access.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && rm -f /var/log/httpd/access.log && mkfifo /var/log/httpd/access.log && while true; do cat /var/log/httpd/access.log; done']
volumes:
- PlacementLogs:/var/log/
- PlacementLogs:/var/log
placement_logs:
start_order: 2
image: {get_attr: [RoleParametersValue, value, ContainerPlacementImage]}
@ -74,7 +74,7 @@ outputs:
test: '[ -p /var/log/placement/placement-api.log ]'
command: ['/bin/bash', '-c', 'mkdir -p /var/log/placement && rm -f /var/log/placement/placement-api.log && mkfifo /var/log/placement/placement-api.log && while true; do cat /var/log/placement/placement-api.log; done']
volumes:
- PlacementLogs:/var/log/
- PlacementLogs:/var/log
host_prep_tasks:
description: Extra ansible tasks needed for logging to files in the host.
value: null

View File

@ -62,7 +62,7 @@ outputs:
- - '/var/lib/kolla/config_files/src-ceph'
- - 'ro'
- /dev:/dev
- /run/:/run/
- /run:/run
- /sys:/sys
- /lib/modules:/lib/modules:ro
- /var/lib/manila:/var/lib/manila:z

View File

@ -426,7 +426,7 @@ outputs:
- /var/lib/containers/storage/overlay-containers:/var/lib/containers/storage/overlay-containers:ro
- /var/lib/config-data/ansible-generated/collectd:/var/lib/kolla/config_files/src:ro
- /var/log/containers/collectd:/var/log/collectd:rw,z
- /var/run/:/var/run:rw
- /var/run:/var/run:rw
- /sys/fs/cgroup:/sys/fs/cgroup:ro
environment:
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS

View File

@ -1402,7 +1402,7 @@ outputs:
- /var/lib/kolla/config_files/nova_compute_wait_for_compute_service.json:/var/lib/kolla/config_files/config.json:ro
- /var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro
- /var/log/containers/nova:/var/log/nova
- /var/lib/container-config-scripts/:/container-config-scripts/
- /var/lib/container-config-scripts:/container-config-scripts
user: nova
environment:
KOLLA_CONFIG_STRATEGY: COPY_ALWAYS

View File

@ -374,7 +374,7 @@ outputs:
step_config: 'include tripleo::profile::base::rabbitmq'
config_image: *rabbitmq_config_image
volumes:
- /var/lib/config-data/puppet-generated/rabbitmq/etc/rabbitmq/:/etc/rabbitmq/:ro
- /var/lib/config-data/puppet-generated/rabbitmq/etc/rabbitmq:/etc/rabbitmq:ro
- /var/lib/rabbitmq:/var/lib/rabbitmq
metadata_settings:
if:

View File

@ -267,7 +267,7 @@ outputs:
step_config: 'include tripleo::profile::base::rabbitmq'
config_image: *rabbitmq_config_image
volumes:
- /var/lib/config-data/puppet-generated/rabbitmq/etc/rabbitmq/:/etc/rabbitmq/:ro
- /var/lib/config-data/puppet-generated/rabbitmq/etc/rabbitmq:/etc/rabbitmq:ro
- /var/lib/rabbitmq:/var/lib/rabbitmq:z
metadata_settings:
if:

View File

@ -267,7 +267,7 @@ outputs:
step_config: 'include tripleo::profile::base::rabbitmq'
config_image: *rabbitmq_config_image
volumes:
- /var/lib/config-data/puppet-generated/rabbitmq/etc/rabbitmq/:/etc/rabbitmq/:ro
- /var/lib/config-data/puppet-generated/rabbitmq/etc/rabbitmq:/etc/rabbitmq:ro
- /var/lib/rabbitmq:/var/lib/rabbitmq:z
metadata_settings:
if:

View File

@ -142,7 +142,7 @@ outputs:
volumes:
list_concat:
- {get_attr: [ContainersCommon, volumes]}
- - /run/:/run/
- - /run:/run
- /var/lib/kolla/config_files/unbound.json:/var/lib/kolla/config_files/config.json:ro
- /var/log/containers/unbound:/var/log/unbound:z
- /var/lib/config-data/ansible-generated/unbound:/var/lib/kolla/config_files/src:ro

View File

@ -637,6 +637,17 @@ def validate_docker_service_mysql_usage(filename, tpl):
return 0
def validate_common_service(filename, tpl):
# NOTE(bogdando): it doesn't always detect volumes in Heat funcs, like:
# <func>: [{<func>: [FooCommon, volumes]}, [<listed_volumes>]]
if 'outputs' not in tpl:
print('ERROR: missing outputs for filename: %s' % filename)
return 1
if validate_ct_volumes(tpl['outputs']):
return 1
return 0
def validate_docker_service(filename, tpl):
if 'outputs' in tpl and 'role_data' in tpl['outputs']:
if 'value' not in tpl['outputs']['role_data']:
@ -644,6 +655,10 @@ def validate_docker_service(filename, tpl):
% filename)
return 1
role_data = tpl['outputs']['role_data']['value']
if validate_ct_volumes(role_data):
return 1
if list(role_data.keys()) == ['map_merge']:
merged_role_data = {}
for part in role_data['map_merge']:
@ -703,8 +718,6 @@ def validate_docker_service(filename, tpl):
print('ERROR: %s should not be in puppet_config section.'
% key)
return 1
if validate_ct_volumes(puppet_config.get('volumes')):
return 1
for key in REQUIRED_DOCKER_PUPPET_CONFIG_SECTIONS:
if key not in puppet_config:
print('ERROR: %s is required in puppet_config for %s.'
@ -742,8 +755,6 @@ def validate_docker_service(filename, tpl):
print('ERROR: bootstrap_host_exec needs to run '
'as the root user.')
return 1
if validate_ct_volumes(container.get('volumes')):
return 1
if 'upgrade_tasks' in role_data and role_data['upgrade_tasks']:
if (validate_upgrade_tasks(role_data['upgrade_tasks']) or
@ -760,36 +771,53 @@ def validate_docker_service(filename, tpl):
return 0
def validate_ct_volumes(volumes):
'''Ensure we don't have any trailing "/" in the volume'''
if not volumes:
return 0
if isinstance(volumes, list):
# Plain list without much complications
for vol in volumes:
if isinstance(vol, dict):
# Avoid 'if'
continue
vol_def = vol.split(':')
if vol_def[0][-1] == '/' or vol_def[1][-1] == '/':
print('ERROR: trailing "/" detected for {}'.format(vol))
return 1
return 0
ret = 0
if isinstance(volumes, dict):
# We probably face a list_concat thing. Clean and re-run!
# First avoid the get_attr.
if 'get_attr' in list(volumes.keys()):
def validate_ct_volumes(data):
'''Ensure we don't have any trailing "/" in data for volumes'''
def check_volumes(volumes):
if not volumes:
return 0
if 'list_concat' in list(volumes.keys()):
for vol in volumes['list_concat']:
result = 0
if isinstance(volumes, list):
for vol in volumes:
if isinstance(vol, dict):
# Avoid 'if', 'get_*' etc
continue
ret += validate_ct_volumes(vol)
return ret
print('ERROR: unknown "volumes" type: {}'.format(volumes))
return 1
elif isinstance(vol, list):
for item in vol:
result += check_volumes(item)
elif isinstance(vol, str):
vol_def = vol.split(':')
try:
if vol_def[0][-1] == '/' or vol_def[1][-1] == '/':
print('ERROR: trailing "/" detected'
' for {}'.format(vol))
return 1
except IndexError:
# Not a volume definition, ignore it
continue
elif isinstance(volumes, dict):
# Step into 'list_concat', 'map_*' etc.
for item in volumes.values():
result += check_volumes(item)
return result
if not data:
return 0
result = 0
for _, item in enumerate(data):
if isinstance(data[item], dict):
if 'volumes' not in data[item]:
result += validate_ct_volumes(data[item])
else:
result += check_volumes(data[item]['volumes'])
elif isinstance(item, list):
if 'volumes' not in item:
result += validate_ct_volumes(item)
else:
result += check_volumes(item.index('volumes'))
else:
continue
return result
def validate_docker_logging_template(filename, tpl):
@ -803,6 +831,8 @@ def validate_docker_logging_template(filename, tpl):
print('ERROR: The file %s is missing the following output(s):'
' %s' % (filename, ', '.join(missing_entries)))
return 1
if validate_ct_volumes(tpl['outputs']):
return 1
return 0
@ -1144,9 +1174,11 @@ def validate(filename, param_map):
if re.search(r'^\.\/deployment\/logging\/(files|stdout)\/', filename):
retval |= validate_docker_logging_template(filename, tpl)
elif VALIDATE_DOCKER_OVERRIDE.get(filename, False) or (
re.search(r'^\.\/deployment\/.+-container-puppet.yaml$', filename) and
re.search(r'^\.\/deployment\/.+-container(-puppet)*.yaml$', filename) and
VALIDATE_DOCKER_OVERRIDE.get(filename, True)):
retval |= validate_docker_service(filename, tpl)
elif re.search(r'^\.\/deployment\/.+-common.*.yaml$', filename):
retval |= validate_common_service(filename, tpl)
if filename.endswith('hyperconverged-ceph.yaml'):
retval |= validate_hci_compute_services_default(filename, tpl)