Implement algorithm for reapply evaluation priority

Implement algorithm to determine app priorities for reapply evaluation.
Use information provided in metadata to create a directional graph.
Detect cycles and abort.
Unit tests added.

Tests: AIO-SX, AIO-DX
Apps are correctly ordered for reapply evaluation.
Applications reapply order: [u'cert-manager', 'rook-ceph-apps',
'platform-integ-apps', 'oidc-auth-apps', u'stx-openstack']

Story: 2007960
Task: 41781

Signed-off-by: Dan Voiculeasa <dan.voiculeasa@windriver.com>
Change-Id: I375a90b746a0ff4c970305a26c2e3e061b14454e

sysinv/sysinv/sysinv/sysinv/common/constants.py

@@ -1596,6 +1596,7 @@ APP_METADATA_PLATFORM_MANAGED_APPS = 'platform_managed_apps_list'
 APP_METADATA_DESIRED_STATE = 'desired_state'
 APP_METADATA_DESIRED_STATES = 'desired_states'
 APP_METADATA_FORBIDDEN_MANUAL_OPERATIONS = 'forbidden_manual_operations'
+APP_METADATA_ORDERED_APPS = 'ordered_apps'
 
 APP_EVALUATE_REAPPLY_TYPE_HOST_ADD = 'host-add'
 APP_EVALUATE_REAPPLY_TYPE_HOST_DELETE = 'host-delete'

sysinv/sysinv/sysinv/sysinv/conductor/kube_app.py

@@ -82,6 +82,7 @@ ARMADA_LOCK_PLURAL = 'locks'
 ARMADA_LOCK_NAME = 'lock'
 
 LOCK_NAME_APP_REAPPLY = 'app_reapply'
+LOCK_NAME_PROCESS_APP_METADATA = 'process_app_metadata'
 
 
 # Helper functions
@@ -1888,6 +1889,200 @@ class AppOperator(object):
         lifecycle_op.app_lifecycle_actions(context, conductor_obj, self, app, hook_info)
 
     @staticmethod
+    def recompute_app_evaluation_order(apps_metadata_dict):
+        """ Get the order of app reapplies based on dependencies
+
+        The following algorithm uses these concepts:
+        Root apps are apps that have no dependency.
+        Chain depth for an app is the number of apps that form the longest
+        chain ending in the current app.
+
+        Main logic:
+        Compute reverse graph (after_apps).
+        Determine root apps.
+        Detect cycles and abort.
+        Compute the longest dependency chain.
+        Traverse again to populate ordered list.
+
+        Assumptions:
+        In theory there is one or few root apps that are dominant vertices.
+        Other than the dominant vertices, there are very sparse vertices with
+        a degree more than one, most of the vertices are either leaves or
+        isolated.
+        Chain depth is usually 0 or 1, few apps have a chain depth of 2, 3, 4
+        The structure is a sparse digraph, or multiple separate sparse digraphs
+        with a total number of vertices equal to the number of apps.
+
+        Complexity analysis:
+        Spatial complexity O(V+E)
+        Cycle detection: O(V+E)
+
+        After cycle detection the graph is a DAG.
+        For computing the chain depth and final traversal a subgraph may be
+        revisited. Complexity would be O(V*E).
+
+        Let k = number of apps with a vertex that have the in degree > 1 and
+        that are not leaf apps. We can bind k to be 0<=k<=10000, shall we reach
+        that app number.
+
+        Each node and each vertex will be visited once O(V+E) (root apps
+        + vertex to leaf).
+        Only k nodes will trigger a revisit of a subset of vertices (k * O(E)).
+
+        Complexity now becomes O(V+(k+1)*E) = O(V+E)
+
+        Limitations:
+        If an app(current) depends only on non-existing apps, then
+        current app will not be properly ordered. It will not be present in
+        the ordered list before other apps based on it.
+        If an app(current) depends only on non platform managed apps, then
+        current app will not be properly ordered. It will not be present in
+        the ordered list before other apps based on it.
+
+        :param: apps_metadata_dict dictionary containing parsed and processed
+                metadata collection
+
+        :return: Sorted list containing the app reapply order.
+        """
+        # Apps directly after current
+        after_apps = {}
+
+        # Remember the maximum depth
+        chain_depth = {}
+
+        # Used to detect cycles
+        cycle_depth = {}
+
+        # Used for second traversal when populating ordered list
+        traverse_depth = {}
+
+        # Final result
+        ordered_apps = []
+        apps_metadata_dict[constants.APP_METADATA_ORDERED_APPS] = ordered_apps
+
+        # Initialize structures
+        for app_name in apps_metadata_dict[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            after_apps[app_name] = []
+            chain_depth[app_name] = 0
+            cycle_depth[app_name] = 0
+            traverse_depth[app_name] = 0
+
+        # For each app remember which apps are directly after
+        for app_name in apps_metadata_dict[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            app_metadata = apps_metadata_dict[constants.APP_METADATA_APPS][app_name]
+            metadata_after = app_metadata.get(constants.APP_METADATA_BEHAVIOR, None)
+
+            if metadata_after is not None:
+                metadata_after = metadata_after.get(constants.APP_METADATA_EVALUATE_REAPPLY, None)
+            if metadata_after is not None:
+                metadata_after = metadata_after.get(constants.APP_METADATA_AFTER, None)
+            if metadata_after is not None:
+                for before_app in metadata_after:
+                    # This one may be a non-existing app, need to initialize
+                    if after_apps.get(before_app, None) is None:
+                        after_apps[before_app] = []
+
+                    # Store information
+                    after_apps[before_app].append(app_name)
+
+                    # Remember that current app is before at least one
+                    chain_depth[app_name] = 1
+                    traverse_depth[app_name] = 1
+
+        # Identify root apps
+        root_apps = []
+        for app_name in apps_metadata_dict[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            if chain_depth.get(app_name, None) == 0:
+                root_apps.append(app_name)
+
+        # Used for cycle detection
+        stack_ = queue.LifoQueue()
+        cycle_checked = {}
+        max_depth = len(apps_metadata_dict[constants.APP_METADATA_PLATFORM_MANAGED_APPS])
+
+        # Detect cycles and abort
+        for app_name in apps_metadata_dict[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            # Skip already checked app
+            if cycle_checked.get(app_name, False) is True:
+                continue
+
+            # Start from this
+            stack_.put(app_name)
+
+            # Reinitialize temporary visited
+            visited = {}
+
+            # Traverse DFS to detect cycles
+            while not stack_.empty():
+                app_name = stack_.get_nowait()
+                visited[app_name] = True
+
+                # Skip already checked app
+                if cycle_checked.get(app_name, False) is True:
+                    continue
+
+                for after in after_apps[app_name]:
+                    cycle_depth[after] = max(cycle_depth[app_name] + 1, cycle_depth[after])
+                    # Detected cycle
+                    if cycle_depth[after] > max_depth:
+                        return ordered_apps
+
+                    stack_.put(after)
+
+            # Remember the temporary visited apps to skip them in the future
+            for r in visited.keys():
+                cycle_checked[r] = True
+
+        # Used for traversal
+        queue_ = queue.Queue()
+
+        # Compute the longest dependency chain starting from root apps
+        for app_name in root_apps:
+            queue_.put(app_name)
+
+        # Traverse similar to BFS to compute the longest dependency chain
+        while not queue_.empty():
+            app_name = queue_.get_nowait()
+            for after in after_apps[app_name]:
+                chain_depth[after] = max(chain_depth[app_name] + 1, chain_depth[after])
+                queue_.put(after)
+
+        # Traverse graph again similar to BFS
+        # Add to ordered list when the correct chain depth is reached
+        found = {}
+        for app_name in root_apps:
+            queue_.put(app_name)
+            found[app_name] = True
+            ordered_apps.append(app_name)
+
+        while not queue_.empty():
+            app_name = queue_.get_nowait()
+
+            for after in after_apps[app_name]:
+                traverse_depth[after] = max(traverse_depth[app_name] + 1, traverse_depth[after])
+
+                # This is the correct depth, add to ordered list
+                if traverse_depth[after] == chain_depth[after]:
+                    # Skip if already added
+                    if found.get(after, False) is True:
+                        continue
+
+                    found[after] = True
+                    ordered_apps.append(after)
+
+                queue_.put(after)
+
+        # Add apps that have dependencies on non-existing apps
+        for app_name in apps_metadata_dict[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            if found.get(app_name, False) is True:
+                continue
+            ordered_apps.append(app_name)
+
+        LOG.info("Applications reapply order: {}".format(ordered_apps))
+        apps_metadata_dict[constants.APP_METADATA_ORDERED_APPS] = ordered_apps
+
+    @staticmethod
+    @cutils.synchronized(LOCK_NAME_PROCESS_APP_METADATA, external=False)
     def update_and_process_app_metadata(apps_metadata_dict, app_name, metadata, overwrite=True):
         """ Update the cached metadata for an app
 
@@ -1920,6 +2115,9 @@ class AppOperator(object):
                 LOG.info("App {} requested to be platform managed"
                          "".format(app_name))
 
+                # Recompute app reapply order
+                AppOperator.recompute_app_evaluation_order(apps_metadata_dict)
+
             # Remember the desired state the app should achieve
             if desired_state is not None:
                 apps_metadata_dict[

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -16,7 +16,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-# Copyright (c) 2013-2020 Wind River Systems, Inc.
+# Copyright (c) 2013-2021 Wind River Systems, Inc.
 #
 
 """Conduct all activity related system inventory.
@@ -224,7 +224,8 @@ class ConductorManager(service.PeriodicService):
 
         self.apps_metadata = {constants.APP_METADATA_APPS: {},
                               constants.APP_METADATA_PLATFORM_MANAGED_APPS: {},
-                              constants.APP_METADATA_DESIRED_STATES: {}}
+                              constants.APP_METADATA_DESIRED_STATES: {},
+                              constants.APP_METADATA_ORDERED_APPS: []}
 
     def start(self):
         self._start()
@@ -11965,22 +11966,30 @@ class ConductorManager(service.PeriodicService):
 
         :returns: list of apps or app names
         """
-        # TODO(dvoicule) reorder apps based on dependencies between them
-        # now make HELM_APP_PLATFORM first to keep backward compatibility
-        ordered_apps = []
-
         try:
-            if filter_active:
-                apps = list(filter(lambda app: app.active, self.dbapi.kube_app_get_all()))
-            else:
-                apps = self.dbapi.kube_app_get_all()
+            # Cached entry: precomputed order of reapply evaluation
+            if name_only and not filter_active:
+                return self.apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+            ordered_apps = []
+            # Start from already ordered list
+            for app_name in self.apps_metadata[constants.APP_METADATA_ORDERED_APPS]:
+                try:
+                    app = self.dbapi.kube_app_get(app_name)
+                except exception.KubeAppNotFound:
+                    continue
+
+                if filter_active and app.active:
+                    ordered_apps.append(app)
+                elif not filter_active:
+                    ordered_apps.append(app)
+
+            LOG.info("Apps reapply order: {}".format([app.name for app in ordered_apps]))
 
-            ordered_apps = sorted(apps, key=lambda x: x.get('name', 'placeholder')
-                                 if x.get('name', 'placeholder') != constants.HELM_APP_PLATFORM else '')
             if name_only:
                 ordered_apps = [app.name for app in ordered_apps]
         except Exception as e:
-            LOG.error("Error while ordering apps for reapply {}".format(e))
+            LOG.error("Error while ordering apps for reapply {}".format(str(e)))
             ordered_apps = []
 
         return ordered_apps

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_1.yaml

@@ -0,0 +1,69 @@
+---
+app_name: app0
+behavior:
+  platform_managed_app: true
+---
+app_name: app1
+behavior:
+  platform_managed_app: true
+---
+app_name: app2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+---
+app_name: app3
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app0
+      - app1
+      - app2
+      - app4
+---
+app_name: app4
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    triggers:
+      - mock: mock
+---
+app_name: app5
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app4
+    triggers:
+      - mock: mock
+---
+app_name: app20
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+---
+app_name: app_2_0
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    triggers:
+      - mock: mock
+---
+app_name: app_2_1
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app_2_0
+---
+app_name: app_2_2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app_2_1

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_cycle_1.yaml

@@ -0,0 +1,30 @@
+---
+app_name: app_last_in_sorted_list
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    triggers:
+      - mock: mock
+---
+app_name: app1
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app_last_in_sorted_list
+      - app3
+---
+app_name: app2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+---
+app_name: app3
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+      - app2

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_cycle_2.yaml

@@ -0,0 +1,22 @@
+---
+app_name: app1
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app3
+---
+app_name: app2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+---
+app_name: app3
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+      - app2

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_cycle_3.yaml

@@ -0,0 +1,7 @@
+---
+app_name: app1
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_non_existing_1.yaml

@@ -0,0 +1,26 @@
+---
+app_name: app0
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app_non_existing_0
+---
+app_name: app1
+behavior:
+  platform_managed_app: true
+---
+app_name: app2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app_non_existing_2
+---
+app_name: app3
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app0
+      - app2

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_non_existing_2.yaml

@@ -0,0 +1,32 @@
+---
+app_name: app0
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app_non_existing_0
+      - app4
+---
+app_name: app1
+behavior:
+  platform_managed_app: true
+---
+app_name: app2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app_non_existing_2
+      - app4
+---
+app_name: app3
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app0
+      - app2
+---
+app_name: app4
+behavior:
+  platform_managed_app: true

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_non_managed_1.yaml

@@ -0,0 +1,24 @@
+---
+app_name: app0
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+---
+app_name: app1
+---
+app_name: app2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+---
+app_name: app3
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app0
+      - app2

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_not_cycle_1_non_managed.yaml

@@ -0,0 +1,21 @@
+---
+app_name: app1
+behavior:
+  evaluate_reapply:
+    after:
+      - app3
+---
+app_name: app2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+---
+app_name: app3
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+      - app2

sysinv/sysinv/sysinv/sysinv/tests/conductor/data/metadata_app_reapply_not_cycle_2.yaml

@@ -0,0 +1,26 @@
+---
+app_name: app0
+behavior:
+  platform_managed_app: true
+---
+app_name: app1
+behavior:
+  evaluate_reapply:
+    after:
+      - app3
+---
+app_name: app2
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+      - app0
+---
+app_name: app3
+behavior:
+  platform_managed_app: true
+  evaluate_reapply:
+    after:
+      - app1
+      - app2

sysinv/sysinv/sysinv/sysinv/tests/conductor/test_kube_app_metadata.py

@@ -0,0 +1,314 @@
+#
+# Copyright (c) 2021 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+"""Test class for Sysinv Kube App Metadata operations."""
+
+import copy
+import os
+import ruamel.yaml as yaml
+
+from sysinv.common import constants
+from sysinv.conductor import kube_app
+from sysinv.conductor import manager
+from sysinv.tests import base
+
+
+class TestKubeAppMetadata(base.TestCase):
+
+    def setUp(self):
+        super(TestKubeAppMetadata, self).setUp()
+
+        # Manager holds apps_metadata dict
+        self.service = manager.ConductorManager('test-host', 'test-topic')
+
+    def test_reapply_order_computation(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_1.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        before_apps = {}
+        self._compute_before_apps(mock_apps_metadata, before_apps)
+
+        # All apps are present in the ordered list
+        for app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            self.assertTrue(app in ordered_list)
+
+        # All apps are present only once
+        self.assertEqual(len(ordered_list),
+                         len(mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]))
+
+        # All apps have the constrains satisfied
+        for app in before_apps.keys():
+            for before in before_apps[app]:
+                self.assertTrue(ordered_list.index(before) < ordered_list.index(app))
+
+    def test_reapply_non_existing_1(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_non_existing_1.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        before_apps = {}
+        self._compute_before_apps(mock_apps_metadata, before_apps)
+
+        # All apps are present in the ordered list
+        for app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            self.assertTrue(app in ordered_list)
+
+        # All apps are present only once
+        self.assertEqual(len(ordered_list),
+                         len(mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]))
+
+        self.skipTest("Corner case not implemented -> "
+                      "app based only on non-existing")
+
+        # All apps have the constrains satisfied
+        for app in before_apps.keys():
+            for before in before_apps[app]:
+                # Skip non-existing
+                if app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS] and \
+                        before in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+                    self.assertTrue(ordered_list.index(before) < ordered_list.index(app))
+
+    def test_reapply_non_existing_2(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_non_existing_2.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        before_apps = {}
+        self._compute_before_apps(mock_apps_metadata, before_apps)
+
+        # All apps are present in the ordered list
+        for app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            self.assertTrue(app in ordered_list)
+
+        # All apps are present only once
+        self.assertEqual(len(ordered_list),
+                         len(mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]))
+
+        # All apps have the constrains satisfied
+        for app in before_apps.keys():
+            for before in before_apps[app]:
+                # Skip non-existing
+                if app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS] and \
+                        before in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+                    self.assertTrue(ordered_list.index(before) < ordered_list.index(app))
+
+    def test_reapply_non_managed_1(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_non_managed_1.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        before_apps = {}
+        self._compute_before_apps(mock_apps_metadata, before_apps)
+
+        # All apps are present in the ordered list
+        for app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            self.assertTrue(app in ordered_list)
+
+        # All apps are present only once
+        self.assertEqual(len(ordered_list),
+                         len(mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]))
+
+        self.skipTest("Corner case not implemented -> "
+                      "app based only on non platform managed")
+
+        # All apps have the constrains satisfied
+        for app in before_apps.keys():
+            for before in before_apps[app]:
+                # Skip non-existing
+                if app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS] and \
+                        before in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+                    self.assertTrue(ordered_list.index(before) < ordered_list.index(app))
+
+    def test_reapply_not_cycle_1(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_not_cycle_1_non_managed.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        before_apps = {}
+        self._compute_before_apps(mock_apps_metadata, before_apps)
+
+        # All apps are present in the ordered list
+        for app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            self.assertTrue(app in ordered_list)
+
+        # All apps are present only once
+        self.assertEqual(len(ordered_list),
+                         len(mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]))
+
+        self.skipTest("Corner case not implemented -> "
+                      "app based only on non platform managed")
+
+        # All apps have the constrains satisfied
+        for app in before_apps.keys():
+            for before in before_apps[app]:
+                # Skip non-existing
+                if app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS] and \
+                        before in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+                    self.assertTrue(ordered_list.index(before) < ordered_list.index(app))
+
+    def test_reapply_not_cycle_2(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_not_cycle_2.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        before_apps = {}
+        self._compute_before_apps(mock_apps_metadata, before_apps)
+
+        # All apps are present in the ordered list
+        for app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            self.assertTrue(app in ordered_list)
+
+        # All apps are present only once
+        self.assertEqual(len(ordered_list),
+                         len(mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]))
+
+        # All apps have the constrains satisfied
+        for app in before_apps.keys():
+            for before in before_apps[app]:
+                # Skip non-existing
+                if app in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS] and \
+                        before in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+                    self.assertTrue(ordered_list.index(before) < ordered_list.index(app))
+
+    def _compute_before_apps(self, mock_apps_metadata, before_apps):
+        # Initialize structures
+        for app_name in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            before_apps[app_name] = []
+
+        # For each app remember which apps are before
+        for app_name in mock_apps_metadata[constants.APP_METADATA_PLATFORM_MANAGED_APPS]:
+            app_metadata = mock_apps_metadata[constants.APP_METADATA_APPS][app_name]
+            metadata_after = app_metadata.get(constants.APP_METADATA_BEHAVIOR, None)
+
+            if metadata_after is not None:
+                metadata_after = metadata_after.get(constants.APP_METADATA_EVALUATE_REAPPLY, None)
+            if metadata_after is not None:
+                metadata_after = metadata_after.get('after', None)
+            if metadata_after is not None:
+                for before_app in metadata_after:
+                    # Append to apps that are before
+                    before_apps[app_name].append(before_app)
+
+    def test_reapply_order_cycle_detection_1(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_cycle_1.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        self.assertEqual(ordered_list, [])
+
+    def test_reapply_order_cycle_detection_2(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_cycle_2.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        self.assertEqual(ordered_list, [])
+
+    def test_reapply_order_cycle_detection_3(self):
+        # Temporary copy
+        mock_apps_metadata = copy.deepcopy(self.service.apps_metadata)
+
+        yaml_file = os.path.join(os.path.dirname(__file__),
+                                 "data", "metadata_app_reapply_cycle_3.yaml")
+        with open(yaml_file, 'r') as f:
+            metadata_collection = yaml.safe_load_all(f)
+
+            for metadata in metadata_collection:
+                kube_app.AppOperator.update_and_process_app_metadata(mock_apps_metadata,
+                                                                     metadata['app_name'],
+                                                                     metadata)
+
+        ordered_list = mock_apps_metadata[constants.APP_METADATA_ORDERED_APPS]
+
+        self.assertEqual(ordered_list, [])