Browse Source

yamlgroup: add regex match; exclude puppet4 for arm64 mirrors

Two related changes that need to go together because we test with the
production groups.yaml.

Confusingly, there are arm64 PC1 puppet repos, and it contains a bunch
of things that it turns out are the common java parts only.  The
puppet-agent package is not available, and it doesn't seem like it
will be [1].  I think this means we can not run puppet4 on our arm64
xenial ci hosts.

The problem is the mirrors have been updated to puppet4 -- runs are
now breaking on the arm mirrors because they don't have puppet-agent
packages.  It seems all we can really do at this point is contine to
run them on puppet3.

This is hard (impossible?) to express with a fnmatch in the existing
yamlgroups syntax.  We could do something like list all the mirror
hosts and use anchors etc, but we have to keep that maintained.  Add
an feature to the inventory plugin that if the list entry starts with
a ^ it is considered a full regex and passed to re.match.  This
allows us to write more complex matchers where required -- in this
case the arm64 ci mirror hosts are excluded from the puppet4 group.

Testing is updated.

[1] https://groups.google.com/forum/#!msg/puppet-dev/iBMYJpvhaWM/WTGmJvXxAgAJ

Change-Id: I828e0c524f8d5ca866786978486bc04829464b47
master
Ian Wienand 1 month ago
parent
commit
4abd0a3184

+ 1
- 1
inventory/groups.yaml View File

@@ -175,7 +175,7 @@ groups:
175 175
     - logstash-worker[0-9]*.open*.org
176 176
     - logstash[0-9]*.open*.org
177 177
     - mirror-update[0-9]*.open*.org
178
-    - mirror[0-9]*.*.*.open*.org
178
+    - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.open.*\.org
179 179
     - openstackid[0-9]*.openstack.org
180 180
     - openstackid-dev[0-9]*.openstack.org
181 181
     - paste[0-9]*.open*.org

+ 5
- 0
playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml View File

@@ -39,6 +39,11 @@ results:
39 39
     - puppet
40 40
     - puppet4
41 41
 
42
+  mirror01.lon1.linaro-london.openstack.org:
43
+    - afs-client
44
+    - mirror
45
+    - puppet
46
+
42 47
   mirror-update01.openstack.org:
43 48
     - afsadmin
44 49
     - puppet

+ 2
- 2
playbooks/roles/install-ansible/files/inventory_plugins/test_yamlgroup.py View File

@@ -37,7 +37,7 @@ class TestInventory(testtools.TestCase):
37 37
 
38 38
         results_yaml = os.path.join(FIXTURE_DIR, 'results.yaml')
39 39
         with open(results_yaml) as f:
40
-            results = yaml.load(f)
40
+            results = yaml.load(f, Loader=yaml.FullLoader)
41 41
             results = results['results']
42 42
 
43 43
         # Build the inventory list.  This is a list of Host objects
@@ -65,7 +65,7 @@ class TestInventory(testtools.TestCase):
65 65
         # real-life, which gets the groups into the config object
66 66
         path = os.path.join(FIXTURE_DIR, 'groups.yaml')
67 67
         with open(path) as f:
68
-            config_groups = yaml.load(f)
68
+            config_groups = yaml.load(f, Loader=yaml.FullLoader)
69 69
             config_groups = config_groups['groups']
70 70
         im = InventoryModule()
71 71
         im._read_config_data = mock.MagicMock()

+ 15
- 2
playbooks/roles/install-ansible/files/inventory_plugins/yamlgroup.py View File

@@ -3,6 +3,7 @@
3 3
 
4 4
 import fnmatch
5 5
 import os
6
+import re
6 7
 
7 8
 from ansible.parsing.yaml.objects import AnsibleMapping
8 9
 from ansible.plugins.inventory import BaseFileInventoryPlugin
@@ -28,7 +29,10 @@ DOCUMENTATION = '''
28 29
           - section: inventory_plugin_yaml
29 30
             key: yaml_valid_extensions
30 31
       groups:
31
-        description: dict with group name as key and list of fnmatch patterns
32
+        description: |
33
+          dict with group name as key. If the list item starts with a
34
+          ^ it will be considered a regex pattern (i.e. passed to
35
+          re.match), otherwise it is considered a fnmatch pattern.
32 36
         type: dict
33 37
         default: {}
34 38
 '''
@@ -38,6 +42,7 @@ groups:
38 42
   amazing:
39 43
     - fullhost.example.com
40 44
     - amazing*
45
+    - ^regex.*pattern
41 46
 '''
42 47
 
43 48
 
@@ -75,8 +80,16 @@ class InventoryModule(BaseFileInventoryPlugin):
75 80
                 # failing.
76 81
                 if isinstance(candidate, AnsibleMapping):
77 82
                     candidate = list(candidate.keys())[0]
83
+
84
+                # Starts with ^ means it is already a regex.
85
+                # Otherwise it's a fnmatch compatible string; use it's
86
+                # helper to turn that into a regex so we have a common
87
+                # match below.
88
+                if not candidate.startswith('^'):
89
+                    candidate = fnmatch.translate(candidate)
90
+
78 91
                 for existing in self.inventory.hosts.values():
79
-                    if fnmatch.fnmatch(existing.get_name(), candidate):
92
+                    if re.match(candidate, existing.get_name()):
80 93
                         found_groups.setdefault(group, [])
81 94
                         found_groups[group].append(existing)
82 95
 

Loading…
Cancel
Save