Browse Source

Defend against uninitialized fields in k8s objects

Much of these may be unnecessary, but this code was adapted from
go code which handles uninitialized values better via "zero values",
also the k8s python client docs show most of these fields as
"optional".

Hence, initializing leaf values in these model objects to avoid
further surprises.

Change-Id: Ib646b56dfe1ff83f0ecbedaf73fcde8ffa2be0cf
Sean Eagan 2 months ago
parent
commit
623c5056d8
1 changed files with 47 additions and 35 deletions
  1. 47
    35
      armada/handlers/wait.py

+ 47
- 35
armada/handlers/wait.py View File

@@ -463,31 +463,33 @@ class DeploymentWait(ControllerWait):
463 463
         spec = deployment.spec
464 464
         status = deployment.status
465 465
 
466
-        if deployment.metadata.generation <= status.observed_generation:
466
+        gen = deployment.metadata.generation or 0
467
+        observed_gen = status.observed_generation or 0
468
+        if gen <= observed_gen:
467 469
             cond = self._get_resource_condition(status.conditions,
468 470
                                                 'Progressing')
469
-            if cond and cond.reason == 'ProgressDeadlineExceeded':
471
+            if cond and (cond.reason or '') == 'ProgressDeadlineExceeded':
470 472
                 msg = "deployment {} exceeded its progress deadline"
471 473
                 return ("", False, msg.format(name))
472 474
 
473
-            if spec.replicas and status.updated_replicas < spec.replicas:
475
+            replicas = spec.replicas or 0
476
+            updated_replicas = status.updated_replicas or 0
477
+            available_replicas = status.available_replicas or 0
478
+            if updated_replicas < replicas:
474 479
                 msg = ("Waiting for deployment {} rollout to finish: {} out "
475 480
                        "of {} new replicas have been updated...")
476
-                return (msg.format(name, status.updated_replicas,
477
-                                   spec.replicas), False)
481
+                return (msg.format(name, updated_replicas, replicas), False)
478 482
 
479
-            if status.replicas > status.updated_replicas:
483
+            if replicas > updated_replicas:
480 484
                 msg = ("Waiting for deployment {} rollout to finish: {} old "
481 485
                        "replicas are pending termination...")
482
-                pending = status.replicas - status.updated_replicas
486
+                pending = replicas - updated_replicas
483 487
                 return (msg.format(name, pending), False)
484 488
 
485
-            if not self._is_min_ready(status.available_replicas,
486
-                                      status.updated_replicas):
489
+            if not self._is_min_ready(available_replicas, updated_replicas):
487 490
                 msg = ("Waiting for deployment {} rollout to finish: {} of {} "
488 491
                        "updated replicas are available, with min_ready={}")
489
-                return (msg.format(name, status.available_replicas,
490
-                                   status.updated_replicas,
492
+                return (msg.format(name, available_replicas, updated_replicas,
491 493
                                    self.min_ready.source), False, None)
492 494
             msg = "deployment {} successfully rolled out\n"
493 495
             return (msg.format(name), True)
@@ -516,20 +518,24 @@ class DaemonSetWait(ControllerWait):
516 518
                 msg.format(spec.update_strategy.type,
517 519
                            ROLLING_UPDATE_STRATEGY_TYPE))
518 520
 
519
-        if daemon.metadata.generation <= status.observed_generation:
520
-            if (status.updated_number_scheduled <
521
-                    status.desired_number_scheduled):
521
+        gen = daemon.metadata.generation or 0
522
+        observed_gen = status.observed_generation or 0
523
+        updated_number_scheduled = status.updated_number_scheduled or 0
524
+        desired_number_scheduled = status.desired_number_scheduled or 0
525
+        number_available = status.number_available or 0
526
+        if gen <= observed_gen:
527
+            if (updated_number_scheduled < desired_number_scheduled):
522 528
                 msg = ("Waiting for daemon set {} rollout to finish: {} out "
523 529
                        "of {} new pods have been updated...")
524
-                return (msg.format(name, status.updated_number_scheduled,
525
-                                   status.desired_number_scheduled), False)
530
+                return (msg.format(name, updated_number_scheduled,
531
+                                   desired_number_scheduled), False)
526 532
 
527
-            if not self._is_min_ready(status.number_available,
528
-                                      status.desired_number_scheduled):
533
+            if not self._is_min_ready(number_available,
534
+                                      desired_number_scheduled):
529 535
                 msg = ("Waiting for daemon set {} rollout to finish: {} of {} "
530 536
                        "updated pods are available, with min_ready={}")
531
-                return (msg.format(name, status.number_available,
532
-                                   status.desired_number_scheduled,
537
+                return (msg.format(name, number_available,
538
+                                   desired_number_scheduled,
533 539
                                    self.min_ready.source), False)
534 540
 
535 541
             msg = "daemon set {} successfully rolled out"
@@ -552,39 +558,45 @@ class StatefulSetWait(ControllerWait):
552 558
         spec = sts.spec
553 559
         status = sts.status
554 560
 
555
-        if spec.update_strategy.type != ROLLING_UPDATE_STRATEGY_TYPE:
561
+        update_strategy_type = spec.update_strategy.type or ''
562
+        if update_strategy_type != ROLLING_UPDATE_STRATEGY_TYPE:
556 563
             msg = ("Assuming non-readiness for strategy type {}, can only "
557 564
                    "determine for {}")
558 565
 
559 566
             raise armada_exceptions.WaitException(
560
-                msg.format(spec.update_strategy.type,
561
-                           ROLLING_UPDATE_STRATEGY_TYPE))
567
+                msg.format(update_strategy_type, ROLLING_UPDATE_STRATEGY_TYPE))
562 568
 
563
-        if (not status.observed_generation or
564
-                sts.metadata.generation > status.observed_generation):
569
+        gen = sts.metadata.generation or 0
570
+        observed_gen = status.observed_generation or 0
571
+        if (observed_gen == 0 or gen > observed_gen):
565 572
             msg = "Waiting for statefulset spec update to be observed..."
566 573
             return (msg, False)
567 574
 
568
-        if spec.replicas and not self._is_min_ready(status.ready_replicas,
569
-                                                    spec.replicas):
575
+        replicas = spec.replicas or 0
576
+        ready_replicas = status.ready_replicas or 0
577
+        updated_replicas = status.updated_replicas or 0
578
+        current_replicas = status.current_replicas or 0
579
+
580
+        if replicas and not self._is_min_ready(ready_replicas, replicas):
570 581
             msg = ("Waiting for statefulset {} rollout to finish: {} of {} "
571 582
                    "pods are ready, with min_ready={}")
572
-            return (msg.format(name, status.ready_replicas, spec.replicas,
583
+            return (msg.format(name, ready_replicas, replicas,
573 584
                                self.min_ready.source), False)
574 585
 
575
-        if (spec.update_strategy.type == ROLLING_UPDATE_STRATEGY_TYPE and
586
+        if (update_strategy_type == ROLLING_UPDATE_STRATEGY_TYPE and
576 587
                 spec.update_strategy.rolling_update):
577
-            if spec.replicas and spec.update_strategy.rolling_update.partition:
588
+            if replicas and spec.update_strategy.rolling_update.partition:
578 589
                 msg = ("Waiting on partitioned rollout not supported, "
579 590
                        "assuming non-readiness of statefulset {}")
580 591
                 return (msg.format(name), False)
581 592
 
582
-        if status.update_revision != status.current_revision:
593
+        update_revision = status.update_revision or 0
594
+        current_revision = status.current_revision or 0
595
+
596
+        if update_revision != current_revision:
583 597
             msg = ("waiting for statefulset rolling update to complete {} "
584 598
                    "pods at revision {}...")
585
-            return (msg.format(status.updated_replicas,
586
-                               status.update_revision), False)
599
+            return (msg.format(updated_replicas, update_revision), False)
587 600
 
588 601
         msg = "statefulset rolling update complete {} pods at revision {}..."
589
-        return (msg.format(status.current_replicas, status.current_revision),
590
-                True)
602
+        return (msg.format(current_replicas, current_revision), True)

Loading…
Cancel
Save