From d2bc890f30ec330dc44f194585d03262e78d19c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Jeanneret?= Date: Tue, 30 Nov 2021 17:00:31 +0100 Subject: [PATCH] Introduce a new linter for yaml-validate, and correct issues This new linter ensures we don't have any trailing "/" in the container volume definitions. Those trailing "/" may create issues with the containers, for instance for specific mounts such as "/dev"[1]. This patch also takes the opportunity to fix those trailing "/" for the affected files, in order to start on a clean basis. [1] https://launchpad.net/bugs/1950176 Note: the backport is NOT clean: - a service was removed in master and needs some cleanup in wallaby: liquidio-compute-config-container-puppet.yaml - two files had some weird list with empty strings: - neutron-agents-ib-config-container-puppet.yam - neutron-mlnx-agent-container-puppet.yaml Those empty strings have been removed from master apparently. Change-Id: If951f9643d67574c1225301aab7c9e4b0d316b7f Related-Bug: #1950176 (cherry picked from commit 7a99ae23e3804fc24ece379018c68d275dd5d55d) --- .../database/mysql-container-puppet.yaml | 2 +- ...uidio-compute-config-container-puppet.yaml | 8 ++--- .../novajoin/novajoin-container-puppet.yaml | 4 +-- deployment/etcd/etcd-container-puppet.yaml | 2 +- .../horizon/horizon-container-puppet.yaml | 4 +-- .../ironic/ironic-pxe-container-puppet.yaml | 4 +-- .../iscsid/iscsid-container-puppet.yaml | 2 +- .../keystone/keystone-container-puppet.yaml | 2 +- .../metrics/collectd-container-puppet.yaml | 2 +- ...ron-agents-ib-config-container-puppet.yaml | 6 ++-- .../neutron-mlnx-agent-container-puppet.yaml | 3 +- ...utron-ovs-dpdk-agent-container-puppet.yaml | 2 +- .../nova/nova-api-container-puppet.yaml | 2 +- .../nova/nova-compute-container-puppet.yaml | 2 +- .../nova/nova-ironic-container-puppet.yaml | 4 +-- ...ova-migration-target-container-puppet.yaml | 2 +- .../octavia/octavia-api-container-puppet.yaml | 2 +- ...tavia-health-manager-container-puppet.yaml | 2 +- ...octavia-housekeeping-container-puppet.yaml | 2 +- .../octavia-worker-container-puppet.yaml | 2 +- .../placement-api-container-puppet.yaml | 2 +- tools/yaml-validate.py | 36 +++++++++++++++++++ 22 files changed, 65 insertions(+), 32 deletions(-) diff --git a/deployment/database/mysql-container-puppet.yaml b/deployment/database/mysql-container-puppet.yaml index 7c758e4455..d3a273ff20 100644 --- a/deployment/database/mysql-container-puppet.yaml +++ b/deployment/database/mysql-container-puppet.yaml @@ -204,7 +204,7 @@ outputs: list_concat: - *mysql_volumes - - /var/lib/config-data/puppet-generated/mysql/root:/root:rw - - /var/lib/container-config-scripts/:/container-config-scripts/:ro + - /var/lib/container-config-scripts:/container-config-scripts:ro environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS net: host diff --git a/deployment/deprecated/cavium/liquidio-compute-config-container-puppet.yaml b/deployment/deprecated/cavium/liquidio-compute-config-container-puppet.yaml index 544a729725..5369e78921 100644 --- a/deployment/deprecated/cavium/liquidio-compute-config-container-puppet.yaml +++ b/deployment/deprecated/cavium/liquidio-compute-config-container-puppet.yaml @@ -114,10 +114,10 @@ outputs: - - /var/lib/kolla/config_files/liquidio_config.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/liquidio:/var/lib/kolla/config_files/src:ro - - /etc/udev/:/etc/udev/:z - - /usr/lib/udev/:/usr/lib/udev/:z - - /lib/modules/:/lib/modules/:z - - /usr/lib/firmware/liquidio/:/usr/lib/firmware/liquidio/:z + - /etc/udev:/etc/udev:z + - /usr/lib/udev:/usr/lib/udev:z + - /lib/modules:/lib/modules:z + - /usr/lib/firmware/liquidio:/usr/lib/firmware/liquidio:z - /dev:/dev - /run:/run environment: diff --git a/deployment/deprecated/novajoin/novajoin-container-puppet.yaml b/deployment/deprecated/novajoin/novajoin-container-puppet.yaml index e2302aea3e..bc116298e3 100644 --- a/deployment/deprecated/novajoin/novajoin-container-puppet.yaml +++ b/deployment/deprecated/novajoin/novajoin-container-puppet.yaml @@ -211,7 +211,7 @@ outputs: - - /var/lib/kolla/config_files/novajoin_server.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/novajoin:/var/lib/kolla/config_files/src:ro - - /etc/ipa/:/etc/ipa/:ro + - /etc/ipa:/etc/ipa:ro - /etc/novajoin/krb5.keytab:/etc/novajoin/krb5.keytab:ro - /var/log/containers/novajoin:/var/log/novajoin environment: @@ -229,7 +229,7 @@ outputs: - - /var/lib/kolla/config_files/novajoin_notifier.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/novajoin:/var/lib/kolla/config_files/src:ro - - /etc/ipa/:/etc/ipa/:ro + - /etc/ipa:/etc/ipa:ro - /etc/novajoin/krb5.keytab:/etc/novajoin/krb5.keytab:ro - /var/log/containers/novajoin:/var/log/novajoin environment: diff --git a/deployment/etcd/etcd-container-puppet.yaml b/deployment/etcd/etcd-container-puppet.yaml index d5ccc28b88..cc8238d5dc 100644 --- a/deployment/etcd/etcd-container-puppet.yaml +++ b/deployment/etcd/etcd-container-puppet.yaml @@ -176,7 +176,7 @@ outputs: - {get_attr: [ContainersCommon, volumes]} - - /var/lib/etcd:/var/lib/etcd - /var/lib/kolla/config_files/etcd.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/etcd/:/var/lib/kolla/config_files/src:ro + - /var/lib/config-data/puppet-generated/etcd:/var/lib/kolla/config_files/src:ro - if: - internal_tls_enabled - - /etc/pki/tls/certs/etcd.crt:/var/lib/kolla/config_files/src-tls/etc/pki/tls/certs/etcd.crt:ro diff --git a/deployment/horizon/horizon-container-puppet.yaml b/deployment/horizon/horizon-container-puppet.yaml index 01fecbf19d..45531b0b27 100644 --- a/deployment/horizon/horizon-container-puppet.yaml +++ b/deployment/horizon/horizon-container-puppet.yaml @@ -318,8 +318,8 @@ outputs: - /var/lib/config-data/puppet-generated/horizon:/var/lib/kolla/config_files/src:ro - /var/log/containers/horizon:/var/log/horizon:z - /var/log/containers/httpd/horizon:/var/log/httpd:z - - /var/tmp/horizon:/var/tmp/:z - - /var/www/:/var/www/:ro + - /var/tmp/horizon:/var/tmp:z + - /var/www:/var/www:ro - if: - {get_param: EnableInternalTLS} - - /etc/pki/tls/certs/httpd:/etc/pki/tls/certs/httpd:ro diff --git a/deployment/ironic/ironic-pxe-container-puppet.yaml b/deployment/ironic/ironic-pxe-container-puppet.yaml index c2d8de8c6e..c9c2f0e6e3 100644 --- a/deployment/ironic/ironic-pxe-container-puppet.yaml +++ b/deployment/ironic/ironic-pxe-container-puppet.yaml @@ -137,7 +137,7 @@ outputs: - {get_attr: [ContainersCommon, volumes]} - - /var/lib/kolla/config_files/ironic_pxe_tftp.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/ironic:/var/lib/kolla/config_files/src:ro - - /var/lib/ironic:/var/lib/ironic/:shared,z + - /var/lib/ironic:/var/lib/ironic:shared,z - /var/log/containers/ironic:/var/log/ironic:z - /var/log/containers/httpd/ironic-pxe:/var/log/httpd:z environment: @@ -157,7 +157,7 @@ outputs: - {get_attr: [ContainersCommon, volumes]} - - /var/lib/kolla/config_files/ironic_pxe_http.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/ironic:/var/lib/kolla/config_files/src:ro - - /var/lib/ironic:/var/lib/ironic/:shared,z + - /var/lib/ironic:/var/lib/ironic:shared,z - /var/log/containers/ironic:/var/log/ironic:z - /var/log/containers/httpd/ironic-pxe:/var/log/httpd:z environment: diff --git a/deployment/iscsid/iscsid-container-puppet.yaml b/deployment/iscsid/iscsid-container-puppet.yaml index 93588c30c0..6b0408e76a 100644 --- a/deployment/iscsid/iscsid-container-puppet.yaml +++ b/deployment/iscsid/iscsid-container-puppet.yaml @@ -143,7 +143,7 @@ outputs: - {get_attr: [ContainersCommon, volumes]} - - /var/lib/kolla/config_files/iscsid.json:/var/lib/kolla/config_files/config.json:ro - /dev:/dev - - /run/:/run/ + - /run:/run - /sys:/sys - /lib/modules:/lib/modules:ro - /var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro diff --git a/deployment/keystone/keystone-container-puppet.yaml b/deployment/keystone/keystone-container-puppet.yaml index 446f9f5cad..1868542410 100644 --- a/deployment/keystone/keystone-container-puppet.yaml +++ b/deployment/keystone/keystone-container-puppet.yaml @@ -726,7 +726,7 @@ outputs: - {get_attr: [ContainersCommon, volumes]} - {get_attr: [KeystoneLogging, volumes]} - - /var/lib/kolla/config_files/keystone_cron.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/puppet-generated/keystone/:/var/lib/kolla/config_files/src:ro + - /var/lib/config-data/puppet-generated/keystone:/var/lib/kolla/config_files/src:ro environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS step_4: diff --git a/deployment/metrics/collectd-container-puppet.yaml b/deployment/metrics/collectd-container-puppet.yaml index 5a1d7f4fcf..323bc9d326 100644 --- a/deployment/metrics/collectd-container-puppet.yaml +++ b/deployment/metrics/collectd-container-puppet.yaml @@ -753,7 +753,7 @@ outputs: - /var/lib/config-data/puppet-generated/collectd:/var/lib/kolla/config_files/src:ro - /var/log/containers/collectd:/var/log/collectd:rw,z - /var/lib/container-config-scripts:/scripts:ro - - /run/:/run:rw + - /run:/run:rw - /sys/fs/cgroup:/sys/fs/cgroup:ro environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS diff --git a/deployment/neutron/neutron-agents-ib-config-container-puppet.yaml b/deployment/neutron/neutron-agents-ib-config-container-puppet.yaml index 84485ec6a9..3ce86c771f 100644 --- a/deployment/neutron/neutron-agents-ib-config-container-puppet.yaml +++ b/deployment/neutron/neutron-agents-ib-config-container-puppet.yaml @@ -112,11 +112,9 @@ outputs: volumes: list_concat: - {get_attr: [ContainersCommon, volumes]} - - - - /lib/modules:/lib/modules:ro - - /usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro + - - /lib/modules:/lib/modules:ro + - /usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro - /var/lib/config-data/puppet-generated/neutron/etc/neutron:/etc/neutron - - '' environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier} diff --git a/deployment/neutron/neutron-mlnx-agent-container-puppet.yaml b/deployment/neutron/neutron-mlnx-agent-container-puppet.yaml index 12f4df4b46..2e996ecc4c 100644 --- a/deployment/neutron/neutron-mlnx-agent-container-puppet.yaml +++ b/deployment/neutron/neutron-mlnx-agent-container-puppet.yaml @@ -171,9 +171,8 @@ outputs: - /var/lib/kolla/config_files/neutron_mlnx_agent.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro - /lib/modules:/lib/modules:ro - - /usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro + - /usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro - /var/lib/config-data/puppet-generated/neutron/etc/neutron:/etc/neutron - - '' environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier} diff --git a/deployment/neutron/neutron-ovs-dpdk-agent-container-puppet.yaml b/deployment/neutron/neutron-ovs-dpdk-agent-container-puppet.yaml index 21fc289ea3..ee23d18542 100644 --- a/deployment/neutron/neutron-ovs-dpdk-agent-container-puppet.yaml +++ b/deployment/neutron/neutron-ovs-dpdk-agent-container-puppet.yaml @@ -137,7 +137,7 @@ outputs: volumes: - /lib/modules:/lib/modules:ro - /run/openvswitch:/run/openvswitch - - /etc/modules-load.d/:/etc/modules-load.d + - /etc/modules-load.d:/etc/modules-load.d kolla_config: get_attr: [NeutronOvsAgent, role_data, kolla_config] container_config_scripts: diff --git a/deployment/nova/nova-api-container-puppet.yaml b/deployment/nova/nova-api-container-puppet.yaml index cf70b7402e..60156aa367 100644 --- a/deployment/nova/nova-api-container-puppet.yaml +++ b/deployment/nova/nova-api-container-puppet.yaml @@ -636,7 +636,7 @@ outputs: - {get_attr: [NovaApiLogging, volumes]} - - /var/lib/kolla/config_files/nova_wait_for_api_service.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro - - /var/lib/container-config-scripts/:/container-config-scripts/:z + - /var/lib/container-config-scripts:/container-config-scripts:z environment: __OS_DEBUG: yaql: diff --git a/deployment/nova/nova-compute-container-puppet.yaml b/deployment/nova/nova-compute-container-puppet.yaml index d4ef22dfa2..4ab38738ef 100644 --- a/deployment/nova/nova-compute-container-puppet.yaml +++ b/deployment/nova/nova-compute-container-puppet.yaml @@ -1320,7 +1320,7 @@ outputs: volumes: - /var/lib/nova:/var/lib/nova:shared - /var/lib/_nova_secontext:/var/lib/_nova_secontext:shared,z - - /var/lib/container-config-scripts/:/container-config-scripts/:z + - /var/lib/container-config-scripts:/container-config-scripts:z command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_statedir_ownership.py" environment: # NOTE: this should force this container to re-run on each diff --git a/deployment/nova/nova-ironic-container-puppet.yaml b/deployment/nova/nova-ironic-container-puppet.yaml index 763183d4bf..d011672fd1 100644 --- a/deployment/nova/nova-ironic-container-puppet.yaml +++ b/deployment/nova/nova-ironic-container-puppet.yaml @@ -153,7 +153,7 @@ outputs: volumes: - /var/lib/nova:/var/lib/nova:shared - /var/lib/_nova_secontext:/var/lib/_nova_secontext:shared,z - - /var/lib/container-config-scripts/:/container-config-scripts/ + - /var/lib/container-config-scripts:/container-config-scripts command: "/container-config-scripts/pyshim.sh /container-config-scripts/nova_statedir_ownership.py" step_5: nova_compute: @@ -193,7 +193,7 @@ outputs: - /var/lib/kolla/config_files/nova_ironic_wait_for_compute.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro - /var/log/containers/nova:/var/log/nova - - /var/lib/container-config-scripts/:/container-config-scripts/ + - /var/lib/container-config-scripts:/container-config-scripts user: root environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS diff --git a/deployment/nova/nova-migration-target-container-puppet.yaml b/deployment/nova/nova-migration-target-container-puppet.yaml index 3abc53ca6b..284a16d4d9 100644 --- a/deployment/nova/nova-migration-target-container-puppet.yaml +++ b/deployment/nova/nova-migration-target-container-puppet.yaml @@ -173,7 +173,7 @@ outputs: - {get_attr: [ContainersCommon, volumes]} - - /var/lib/kolla/config_files/nova-migration-target.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro - - /etc/ssh/:/host-ssh/:ro + - /etc/ssh:/host-ssh:ro - /run/libvirt:/run/libvirt:shared,z - /var/lib/nova:/var/lib/nova:shared environment: diff --git a/deployment/octavia/octavia-api-container-puppet.yaml b/deployment/octavia/octavia-api-container-puppet.yaml index 402f269c0a..5c71ce2d94 100644 --- a/deployment/octavia/octavia-api-container-puppet.yaml +++ b/deployment/octavia/octavia-api-container-puppet.yaml @@ -314,7 +314,7 @@ outputs: # missing here because we use the same config_volume for all # octavia services, hence the same container image to generate # configuration. - - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia/ + - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia - /var/log/containers/octavia:/var/log/octavia:z - /var/log/containers/httpd/octavia-api:/var/log/httpd:z command: ['/bin/bash', '-c', 'mkdir -p /etc/octavia/conf.d/octavia-api; chown -R octavia:octavia /etc/octavia/conf.d/octavia-api; chown -R octavia:octavia /var/log/octavia'] diff --git a/deployment/octavia/octavia-health-manager-container-puppet.yaml b/deployment/octavia/octavia-health-manager-container-puppet.yaml index 7de27f06d7..3eed7da80d 100644 --- a/deployment/octavia/octavia-health-manager-container-puppet.yaml +++ b/deployment/octavia/octavia-health-manager-container-puppet.yaml @@ -151,7 +151,7 @@ outputs: # missing here because we use the same config_volume for all # octavia services, hence the same container image to generate # configuration. - - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia/:z + - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia:z command: ['/bin/bash', '-c', 'mkdir -p /etc/octavia/conf.d/octavia-health-manager; chown -R octavia:octavia /etc/octavia/conf.d/octavia-health-manager'] step_5: map_merge: diff --git a/deployment/octavia/octavia-housekeeping-container-puppet.yaml b/deployment/octavia/octavia-housekeeping-container-puppet.yaml index d7e578edb9..2a4d1bb787 100644 --- a/deployment/octavia/octavia-housekeeping-container-puppet.yaml +++ b/deployment/octavia/octavia-housekeeping-container-puppet.yaml @@ -122,7 +122,7 @@ outputs: # missing here because we use the same config_volume for all # octavia services, hence the same container image to generate # configuration. - - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia/:z + - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia:z command: ['/bin/bash', '-c', 'mkdir -p /etc/octavia/conf.d/octavia-housekeeping; chown -R octavia:octavia /etc/octavia/conf.d/octavia-housekeeping'] step_5: octavia_housekeeping: diff --git a/deployment/octavia/octavia-worker-container-puppet.yaml b/deployment/octavia/octavia-worker-container-puppet.yaml index 20eb1fffe3..214bf935bf 100644 --- a/deployment/octavia/octavia-worker-container-puppet.yaml +++ b/deployment/octavia/octavia-worker-container-puppet.yaml @@ -110,7 +110,7 @@ outputs: # missing here because we use the same config_volume for all # octavia services, hence the same container image to generate # configuration. - - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia/:z + - /var/lib/config-data/puppet-generated/octavia/etc/octavia:/etc/octavia:z command: ['/bin/bash', '-c', 'mkdir -p /etc/octavia/conf.d/octavia-worker; chown -R octavia:octavia /etc/octavia/conf.d/octavia-worker'] step_5: octavia_worker: diff --git a/deployment/placement/placement-api-container-puppet.yaml b/deployment/placement/placement-api-container-puppet.yaml index bb1c7028c1..6f57ba2ad9 100644 --- a/deployment/placement/placement-api-container-puppet.yaml +++ b/deployment/placement/placement-api-container-puppet.yaml @@ -321,7 +321,7 @@ outputs: - {get_attr: [PlacementLogging, volumes]} - - /var/lib/kolla/config_files/placement_api_wait_for_service.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/placement:/var/lib/kolla/config_files/src:ro - - /var/lib/container-config-scripts/:/container-config-scripts/:z + - /var/lib/container-config-scripts:/container-config-scripts:z environment: __OS_DEBUG: yaql: diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py index 176d3113f0..510cd76c0e 100755 --- a/tools/yaml-validate.py +++ b/tools/yaml-validate.py @@ -776,6 +776,8 @@ def validate_docker_service(filename, tpl): print('ERROR: %s should not be in puppet_config section.' % key) return 1 + if validate_ct_volumes(puppet_config.get('volumes')): + return 1 for key in REQUIRED_DOCKER_PUPPET_CONFIG_SECTIONS: if key not in puppet_config: print('ERROR: %s is required in puppet_config for %s.' @@ -813,6 +815,8 @@ def validate_docker_service(filename, tpl): print('ERROR: bootstrap_host_exec needs to run ' 'as the root user.') return 1 + if validate_ct_volumes(container.get('volumes')): + return 1 if 'upgrade_tasks' in role_data and role_data['upgrade_tasks']: if (validate_upgrade_tasks(role_data['upgrade_tasks']) or @@ -829,6 +833,38 @@ def validate_docker_service(filename, tpl): return 0 +def validate_ct_volumes(volumes): + '''Ensure we don't have any trailing "/" in the volume''' + if not volumes: + return 0 + if isinstance(volumes, list): + # Plain list without much complications + for vol in volumes: + if isinstance(vol, dict): + # Avoid 'if' + continue + vol_def = vol.split(':') + if vol_def[0][-1] == '/' or vol_def[1][-1] == '/': + print('ERROR: trailing "/" detected for {}'.format(vol)) + return 1 + return 0 + + ret = 0 + if isinstance(volumes, dict): + # We probably face a list_concat thing. Clean and re-run! + # First avoid the get_attr. + if 'get_attr' in list(volumes.keys()): + return 0 + if 'list_concat' in list(volumes.keys()): + for vol in volumes['list_concat']: + if isinstance(vol, dict): + continue + ret += validate_ct_volumes(vol) + return ret + print('ERROR: unknown "volumes" type: {}'.format(volumes)) + return 1 + + def validate_docker_logging_template(filename, tpl): if 'outputs' not in tpl: print('ERROR: outputs are missing from: %s' % filename)