Browse Source

Fail wait when no resources found

This removees a fail-safe that allowed releases which did not
contain pods (intentionally) to still succeed after a best effort
to wait for them until timeout.

Now that we have the ability to disable waiting on resource types
`wait.resources` [0], this fail-safe is no longer needed.

Now when resources are not found, armada will fail with a message
for the user to check their `wait.resources` and labels and
configure as needed. This way we can prompt the user to remove
unnecessary waiting from their deployments.

There is also a longer term plan to make these configurations less
often needed [1].

[0]: https://review.openstack.org/#/c/603901/
[1]: https://review.openstack.org/#/c/636440/

Change-Id: I859326470ecba49f2301705409c51312a601e653
changes/11/640511/3
Sean Eagan 2 months ago
parent
commit
3807db1b6e
1 changed files with 15 additions and 15 deletions
  1. 15
    15
      armada/handlers/wait.py

+ 15
- 15
armada/handlers/wait.py View File

@@ -216,22 +216,22 @@ class ResourceWait(ABC):
216 216
 
217 217
             timed_out, modified, unready, found_resources = (
218 218
                 self._watch_resource_completions(timeout=deadline_remaining))
219
-            if not found_resources:
220
-                if self.skip_if_none_found:
221
-                    return
219
+
220
+            if (not found_resources) and self.skip_if_none_found:
221
+                return
222
+
223
+            if timed_out:
224
+                if not found_resources:
225
+                    details = ('None found! Are `wait.labels` correct? Does '
226
+                               '`wait.resources` need to exclude %s?'.format(
227
+                                   self.resource_type))
222 228
                 else:
223
-                    LOG.warn(
224
-                        'Saw no resources for '
225
-                        'resource type=%s, namespace=%s, labels=(%s). Are the '
226
-                        'labels correct?', self.resource_type,
227
-                        self.chart_wait.namespace, self.label_selector)
228
-
229
-            # TODO(seaneagan): Should probably fail here even when resources
230
-            # were not found, at least once we have an option to ignore
231
-            # wait timeouts.
232
-            if timed_out and found_resources:
233
-                error = "Timed out waiting for resources={}".format(
234
-                    sorted(unready))
229
+                    details = ('These {}s were not ready={}'.format(
230
+                        self.resource_type, sorted(unready)))
231
+                error = (
232
+                    'Timed out waiting for {}s (namespace={}, labels=({})). {}'
233
+                    .format(self.resource_type, self.chart_wait.namespace,
234
+                            self.label_selector, details))
235 235
                 LOG.error(error)
236 236
                 raise k8s_exceptions.KubernetesWatchTimeoutException(error)
237 237
 

Loading…
Cancel
Save