openstack
/
tripleo-validations
Code Issues Proposed changes

oslo-config-validator: Removing merged patches 

While developing this, we discovered a lot of inconsistencies across the
openstack projects. These inconsistencies are now fixed, at least on the
master branch so we can now remove them from the exceptions.

This change I70ab87c9bed093cad883b6301b8a09753fc470d9 was merged so the
yaml bug we found in the generator is now fixed across all the branches.
It's now safe to remove the builtin patch.

Also, os_vif now supports list_opts
(I7fc132f917949d147dd6371d54e01a6cfd995aae) so we can use it to template
the vif_plug_(ovs|linux_bridge) sections. As opposed to cinder, there's
no reference section so we need to statically define them.

Change-Id: I6f86ce6b135be9841b5fcae1f940c829327a2db2
This commit is contained in:
David Vallee Delisle 2021-10-04 22:35:35 -04:00
parent 9c62de3d4c
commit 63f881be6e
8 changed files with 129 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
roles/oslo_config_validator/defaults/main.yml
View File

@ -80,10 +80,6 @@ oslo_config_validator_namespaces_config:
- namespace: glance.store
ignored_messages:
- "ERROR:stevedore.extension:Could not load '(glance.store.)*s3(.Store)*': No module named 'boto3'"
# NOTE(dvd): openstack/puppet-glance: https://review.opendev.org/775633
# This setting was removed from puppet registry in Wallaby
# but it was removed from Glance a while ago.
- 'ERROR:root:DEFAULT/enable_v1_api {{ invalid_setting_regex }}'
- namespace: heat.common.config
ignored_messages:
@ -92,15 +88,6 @@ oslo_config_validator_namespaces_config:
# on during the Xena cycle and should hopefully be backported down to train.
- 'ERROR:root:(yaql|(resource_finder_)*cache)/[^\s]+ {{ invalid_setting_regex }}'
- namespace: ironic
ignored_messages:
# NOTE(dvd): openstack/puppet-ironic: https://review.opendev.org/789592
# This setting was removed a while ago but puppet registry was updated only only recently
- 'ERROR:root:pxe/ipxe_enabled {{ invalid_setting_regex }}'
# NOTE(dvd): openstack/puppet-ironic: https://review.opendev.org/790526
# This setting isn't used in ironic since 2013, it was removed in Xena cycle.
- 'ERROR:root:conductor/max_time_interval {{ invalid_setting_regex }}'
- namespace: keystonemiddleware.auth_token
ignored_messages:
- >-
@ -128,10 +115,6 @@ oslo_config_validator_namespaces_config:
value_list:
- "True"
ignored_messages:
# NOTE(dvd): openstack/neutron: https://review.opendev.org/789648
- 'ERROR:root:placement/[^\s]+ {{ invalid_setting_regex }}'
- namespace: cinder
invalid_settings:
- section: DEFAULT
@ -164,13 +147,6 @@ oslo_config_validator_namespaces_config:
- "True"
ignored_messages:
# NOTE(dvd): openstack/puppet-nova: https://review.opendev.org/789633
# This was erroneously categorized in the DEFAULT section but it should have been
# under the vif_plug_ovs section.
- 'ERROR:root:DEFAULT/ovsdb_connection {{ invalid_setting_regex }}'
# NOTE(dvd): openstack/os-vif: https://review.opendev.org/789645
# os_vif had no list_opts entrypoint before Xena.
- 'ERROR:root:vif_plug_ovs/ovsdb_connection {{ invalid_setting_regex }}'
# These settings are used by openstacksdk and not part of oslo_config_opts. They are not taken
# into account by oslo-config-(generator|validator)
- 'ERROR:root:(cinder|service_user)/region_name {{ invalid_setting_regex }}'
@ -196,9 +172,17 @@ oslo_config_validator_service_configs:
config_files:
- path: /etc/nova/nova.conf
namespaces: []
opt_data:
- group_create:
template: os_vif_ovs
group_name: vif_plug_ovs
- group_create:
template: os_vif_linux_bridge
group_name: vif_plug_linux_brige
default_namespaces:
- nova.conf
- keystonemiddleware.auth_token
- os_vif
- oslo.log
- oslo.messaging
- oslo.policy

2
roles/oslo_config_validator/molecule/default/molecule.yml
View File

@ -21,8 +21,6 @@ provisioner:
config_file: etc/nova/nova.conf
config_folder: /var/lib/config-data/puppet-generated/nova_libvirt
validator_out: >-
ERROR:root:DEFAULT/ovsdb_connection not found
ERROR:root:cinder/region_name not found
INFO:root:Ignoring missing option "auth_url" from group "keystone_authtoken" because the group

74
roles/oslo_config_validator/resources/library/generator.yml Normal file
View File

@ -0,0 +1,74 @@
---
options:
DEFAULT:
driver_option: ''
driver_opts: {}
dynamic_group_owner: ''
help: ''
opts:
- advanced: false
choices: []
default: internal
deprecated_for_removal: false
deprecated_opts: []
deprecated_reason: null
deprecated_since: null
dest: internal_service_availability_zone
max: null
metavar: null
min: null
mutable: false
name: internal_service_availability_zone
namespace: nova.conf
positional: false
required: false
sample_default: null
secret: false
short: null
type: string value
os_vif_linux_bridge:
driver_option: ''
driver_opts: {}
dynamic_group_owner: ''
help: ''
opts:
- advanced: false
choices: []
default: false
dest: use_ipv6
help: Use IPv6
max: null
metavar: null
min: null
mutable: false
name: use_ipv6
namespace: os_vif
positional: false
required: false
sample_default: null
secret: false
short: null
type: boolean value
os_vif_ovs:
driver_option: ''
driver_opts: {}
dynamic_group_owner: ''
help: ''
opts:
- advanced: false
choices: []
default: 1500
dest: network_device_mtu
help: MTU setting for network interface.
max: null
metavar: null
min: null
mutable: false
name: network_device_mtu
namespace: os_vif
positional: false
required: false
sample_default: null
secret: false
short: null
type: integer value

9
roles/oslo_config_validator/resources/library/podman
View File

@ -267,6 +267,11 @@ def read_config(config='/test.config.yml'):
return test_config
def mock_generator():
with open('/generator.yml', 'r') as generator:
return "".join(generator.readlines())
def container_list():
test_config = read_config('/test.config.yml')
config_folder = test_config.get('config_folder')
@ -286,7 +291,9 @@ if __name__ == '__main__':
print(dumps([image_get]))
elif "version" in sys.argv[1]:
print("podman version 3.3.1")
elif len(sys.argv) > 1 and sys.argv[2] == "run":
elif "oslo-config-generator" in sys.argv:
print(mock_generator())
elif len(sys.argv) > 2 and sys.argv[2] == "run":
print(container_exec())
elif sys.argv[1] == "container":
container = container_list()

5
roles/oslo_config_validator/resources/playbooks/prepare.yml
View File

@ -26,6 +26,11 @@
content: "{{ test_config }}"
dest: "/test.config.yml"
- name: Saving test_config to file
copy:
src: "{{ playbook_dir }}/../library/generator.yml"
dest: "/generator.yml"
- name: Creating mocked config folder
file:
path: "{{ config_file | dirname }}"

66
roles/oslo_config_validator/tasks/container_run.yml
View File

@ -19,72 +19,6 @@
# this shouldn't be necessary once this proposed change has landed
# and propagated in images
# https://review.opendev.org/c/openstack/oslo.config/+/790883
- name: Applying the most dirty patch in tripleo-validation history
when:
- "'oslo-config-generator' in oslo_command"
vars:
git_apply: >-
git apply --unsafe-paths --directory /usr/lib/python3.*/site-packages/oslo_config/
-p2 -C1 --recount /oslo_config_validation/oslo_config_generator.diff | cat
block:
- name: Generating namespace arguments
set_fact:
generator_patch: |
cat << EOF > /oslo_config_validation/oslo_config_generator.diff
diff --git a/oslo_config/generator.py b/oslo_config/generator.py
index 9845c81..5dd1044 100644
--- a/oslo_config/generator.py
+++ b/oslo_config/generator.py
@@ -52,6 +52,7 @@
import yaml
from oslo_config import cfg
+from oslo_i18n import _message
import stevedore.named # noqa
LOG = logging.getLogger(__name__)
@@ -723,6 +724,18 @@
return output_data
+def i18n_representer(dumper, data):
+ """oslo_i18n yaml representer
+
+ Returns a translated to the default locale string for yaml.safe_dump
+
+ :param dumper: a SafeDumper instance passed by yaml.safe_dump
+ :param data: a oslo_i18n._message.Message instance
+ """
+ serializedData = str(data.translation())
+ return dumper.represent_str(serializedData)
+
+
def _output_machine_readable(groups, output_file, conf):
"""Write a machine readable sample config file
@@ -736,6 +749,7 @@
"""
output_data = _generate_machine_readable_data(groups, conf)
if conf.format_ == 'yaml':
+ yaml.SafeDumper.add_representer(_message.Message, i18n_representer)
output_file.write(yaml.safe_dump(output_data,
default_flow_style=False))
else:
EOF
{{ git_apply }}
{{ oslo_command }}
- name: Sending over the patched script
changed_when: false
copy:
mode: 0777
content: "{{ generator_patch }}"
dest: "{{ oslo_config_validator_work_path }}/oslo_config_generator.sh"
- name: Replacing oslo_command with script
set_fact:
oslo_command: /oslo_config_validation/oslo_config_generator.sh
- name: Printing oslo-config command
when:
- oslo_config_validator_debug | bool

50
roles/oslo_config_validator/tasks/opt_data_format.yml
View File

@ -19,38 +19,46 @@
# mapping in the config file settings. This has to be looped for
# each mapping.
- name: Get list value
become: true
validations_read_ini:
path: "{{ config_file.path }}"
section: "{{ option_data.index_key.section }}"
key: "{{ option_data.index_key.option }}"
register: opt_val
- name: Get list of templatable sections from config file
when: "'index_key' in option_data"
block:
- name: Reading config file to get template
become: true
validations_read_ini:
path: "{{ config_file.path }}"
section: "{{ option_data.index_key.section }}"
key: "{{ option_data.index_key.option }}"
register: opt_val
- name: Printing templatable values
when:
- oslo_config_validator_debug | bool
debug:
var: opt_val
- name: Printing templatable values
when:
- oslo_config_validator_debug | bool
debug:
var: opt_val
- name: Building list of replaceable data
when:
- opt_val.value
- opt_data | length
- "('index_key' in option_data and opt_val.value) or 'group_create' in option_data"
set_fact:
new_sections: >-
{% set section_list = {} %}{%
set templates = [] %}{%
for section in option_data.template_section %}{%
set _ = templates.extend(opt_data.options[section].opts) %}{%
endfor %}{%
if 'separator' in option_data.index_key %}{%
set sections = opt_val.value.split(option_data.index_key.separator) | list %}{%
else %}{%
set sections = [opt_val.value] %}{%
if 'template_section' in option_data %}{%
for section in option_data.template_section %}{%
set _ = templates.extend(opt_data.options[section].opts) %}{%
endfor %}{%
if 'separator' in option_data.index_key %}{%
set sections = opt_val.value.split(option_data.index_key.separator) | list %}{%
else %}{%
set sections = [opt_val.value] %}{%
endif %}{%
elif 'group_create' in option_data %}{%
set _ = templates.extend(opt_data.options[option_data.group_create.template].opts) %}{%
set sections = [option_data.group_create.group_name] %}{%
endif %}{%
for val in sections %}{%
set _ = section_list.__setitem__(val, {"opts": templates }) %}{%
set _ = section_list.__setitem__(val, {"opts": templates }) %}{%
endfor %}{{ section_list }}
- name: Printing generated sections

5
roles/oslo_config_validator/tasks/validation_command.yml
View File

@ -49,8 +49,13 @@
loop_control:
loop_var: option_data
label: >-
{% if 'index_key' in option_data %}
{{ option_data.index_key.section }} /
{{ option_data.index_key.option }}
{% else %}
{{ option_data.group_create.template }} /
{{ option_data.group_create.group_name }}
{% endif %}
- name: Saving new opt_data content
become: true