From 4abd0a3184028a3412667f73d0e6064ae2a56d52 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 10 Apr 2019 16:34:42 +1000 Subject: [PATCH] yamlgroup: add regex match; exclude puppet4 for arm64 mirrors Two related changes that need to go together because we test with the production groups.yaml. Confusingly, there are arm64 PC1 puppet repos, and it contains a bunch of things that it turns out are the common java parts only. The puppet-agent package is not available, and it doesn't seem like it will be [1]. I think this means we can not run puppet4 on our arm64 xenial ci hosts. The problem is the mirrors have been updated to puppet4 -- runs are now breaking on the arm mirrors because they don't have puppet-agent packages. It seems all we can really do at this point is contine to run them on puppet3. This is hard (impossible?) to express with a fnmatch in the existing yamlgroups syntax. We could do something like list all the mirror hosts and use anchors etc, but we have to keep that maintained. Add an feature to the inventory plugin that if the list entry starts with a ^ it is considered a full regex and passed to re.match. This allows us to write more complex matchers where required -- in this case the arm64 ci mirror hosts are excluded from the puppet4 group. Testing is updated. [1] https://groups.google.com/forum/#!msg/puppet-dev/iBMYJpvhaWM/WTGmJvXxAgAJ Change-Id: I828e0c524f8d5ca866786978486bc04829464b47 --- inventory/groups.yaml | 2 +- .../test-fixtures/results.yaml | 5 +++++ .../files/inventory_plugins/test_yamlgroup.py | 4 ++-- .../files/inventory_plugins/yamlgroup.py | 17 +++++++++++++++-- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/inventory/groups.yaml b/inventory/groups.yaml index 58c3be7fec..0edfba0b88 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -175,7 +175,7 @@ groups: - logstash-worker[0-9]*.open*.org - logstash[0-9]*.open*.org - mirror-update[0-9]*.open*.org - - mirror[0-9]*.*.*.open*.org + - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.open.*\.org - openstackid[0-9]*.openstack.org - openstackid-dev[0-9]*.openstack.org - paste[0-9]*.open*.org diff --git a/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml b/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml index 33add16ce5..23f8082c2c 100644 --- a/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml +++ b/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml @@ -39,6 +39,11 @@ results: - puppet - puppet4 + mirror01.lon1.linaro-london.openstack.org: + - afs-client + - mirror + - puppet + mirror-update01.openstack.org: - afsadmin - puppet diff --git a/playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py b/playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py index 7562ddb4fe..f6b18733fe 100644 --- a/playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py +++ b/playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py @@ -37,7 +37,7 @@ class TestInventory(testtools.TestCase): results_yaml = os.path.join(FIXTURE_DIR, 'results.yaml') with open(results_yaml) as f: - results = yaml.load(f) + results = yaml.load(f, Loader=yaml.FullLoader) results = results['results'] # Build the inventory list. This is a list of Host objects @@ -65,7 +65,7 @@ class TestInventory(testtools.TestCase): # real-life, which gets the groups into the config object path = os.path.join(FIXTURE_DIR, 'groups.yaml') with open(path) as f: - config_groups = yaml.load(f) + config_groups = yaml.load(f, Loader=yaml.FullLoader) config_groups = config_groups['groups'] im = InventoryModule() im._read_config_data = mock.MagicMock() diff --git a/playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py b/playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py index 4fb41cae83..1b47315232 100644 --- a/playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py +++ b/playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py @@ -3,6 +3,7 @@ import fnmatch import os +import re from ansible.parsing.yaml.objects import AnsibleMapping from ansible.plugins.inventory import BaseFileInventoryPlugin @@ -28,7 +29,10 @@ DOCUMENTATION = ''' - section: inventory_plugin_yaml key: yaml_valid_extensions groups: - description: dict with group name as key and list of fnmatch patterns + description: | + dict with group name as key. If the list item starts with a + ^ it will be considered a regex pattern (i.e. passed to + re.match), otherwise it is considered a fnmatch pattern. type: dict default: {} ''' @@ -38,6 +42,7 @@ groups: amazing: - fullhost.example.com - amazing* + - ^regex.*pattern ''' @@ -75,8 +80,16 @@ class InventoryModule(BaseFileInventoryPlugin): # failing. if isinstance(candidate, AnsibleMapping): candidate = list(candidate.keys())[0] + + # Starts with ^ means it is already a regex. + # Otherwise it's a fnmatch compatible string; use it's + # helper to turn that into a regex so we have a common + # match below. + if not candidate.startswith('^'): + candidate = fnmatch.translate(candidate) + for existing in self.inventory.hosts.values(): - if fnmatch.fnmatch(existing.get_name(), candidate): + if re.match(candidate, existing.get_name()): found_groups.setdefault(group, []) found_groups[group].append(existing)