Adding the senlin resource

This handle the senlin cluster disable/clean
for disable it will detach the polices from the cluster
and resize the cluster to zero (empty cluster)

for clean it will deatach the policies from cluster
clean the policies, receivers then clean the clusters then clean
the profiles

Change-Id: I09421d55c66d91a747bc786fdd250730c79dfd34

.zuul.yaml

@@ -6,12 +6,14 @@
       # Additional supported projects
       - openstack/designate
       - openstack/octavia
+      - openstack/senlin
     roles:
       - zuul: openstack-infra/devstack
     vars:
         devstack_plugins:
             designate: https://opendev.org/openstack/designate
             octavia: https://opendev.org/openstack/octavia
+            senlin: https://opendev.org/openstack/senlin
         devstack_localrc:
             DISABLE_AMP_IMAGE_BUILD: True
         devstack_local_conf:
@@ -25,6 +27,7 @@
                         cert_manager: local_cert_manager
         devstack_services:
             octavia: true
+            senlin: true
             o-api: true
             o-cw: true
             o-hm: false

ospurge/resources/senlin.py

@@ -0,0 +1,147 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+from ospurge.resources import base
+
+
+class Receivers(base.ServiceResource):
+    ORDER = 99
+
+    def list(self):
+        if not self.cloud.has_service('clustering'):
+            return []
+        return self.cloud.cluster.receivers()
+
+    def delete(self, resource):
+        self.cloud.cluster.delete_receiver(resource['id'])
+
+    @staticmethod
+    def to_str(resource):
+        return "Receiver (id='{}', name='{}')".format(
+            resource['id'], resource['name'])
+
+
+class Policies(base.ServiceResource):
+    ORDER = 100
+    ATTACHED_CLUSTERS = None
+
+    def __init__(self, creds_manager):
+        self.ATTACHED_CLUSTERS = {}
+        super(Policies, self).__init__(creds_manager)
+
+    def populate_bindings(self):
+        if self.ATTACHED_CLUSTERS:
+            return
+
+        for cluster in self.cloud.cluster.clusters():
+            for policy in self.cloud.cluster.cluster_policies(cluster.id):
+                self.append_to_attached_clusters(policy, cluster)
+
+    def append_to_attached_clusters(self, policy, cluster):
+        if policy.policy_id not in self.ATTACHED_CLUSTERS:
+            self.ATTACHED_CLUSTERS[policy.policy_id] = []
+        self.ATTACHED_CLUSTERS[policy.policy_id].append(cluster.id)
+
+    def list(self):
+        if not self.cloud.has_service('clustering'):
+            return []
+        # populate the bindings of the policies
+        self.populate_bindings()
+        return self.cloud.cluster.policies()
+
+    def delete(self, resource):
+        policy_attached_clusters = self.ATTACHED_CLUSTERS.get(
+            resource['id'], [])
+        for attached_cluster_id in policy_attached_clusters:
+            self.cloud.detach_policy_from_cluster(
+                cluster=attached_cluster_id,
+                policy=resource['id'],
+                wait=True
+            )
+
+        self.cloud.cluster.delete_policy(policy=resource['id'])
+
+    def disable(self, resource):
+        policy_attached_clusters = self.ATTACHED_CLUSTERS.get(
+            resource['id'], [])
+        for attached_cluster_id in policy_attached_clusters:
+            self.cloud.cluster.update_cluster_policy(
+                cluster=attached_cluster_id,
+                policy=resource['id'],
+                enabled=False
+            )
+
+    @staticmethod
+    def to_str(resource):
+        return "Policy (id='{}', name='{}')".format(
+            resource['id'], resource['name'])
+
+
+class Clusters(base.ServiceResource):
+    ORDER = 101
+    SPEC = {
+        'min_size': 0,
+        'max_size': 0,
+        'adjustment_type': 'EXACT_CAPACITY',
+        'number': 0
+    }
+
+    def check_prerequisite(self):
+        if not self.cloud.has_service('clustering'):
+            return True
+        # We can't delete till policies, receivers are deleted
+        policy_list_gen = self.cloud.cluster.policies()
+        receiver_list_gen = self.cloud.cluster.receivers()
+        return next(policy_list_gen, None) is None and \
+            next(receiver_list_gen, None) is None
+
+    def list(self):
+        if not self.cloud.has_service('clustering'):
+            return []
+        return self.cloud.cluster.clusters()
+
+    def delete(self, resource):
+        self.cloud.cluster.delete_cluster(cluster=resource['id'])
+
+    def disable(self, resource):
+        self.cloud.cluster.resize_cluster(
+            cluster=resource['id'],
+            **Clusters.SPEC
+        )
+
+    @staticmethod
+    def to_str(resource):
+        return "Cluster (id='{}', name='{}')".format(
+            resource['id'], resource['name'])
+
+
+class Profiles(base.ServiceResource):
+    ORDER = 103
+
+    def check_prerequisite(self):
+        if not self.cloud.has_service('clustering'):
+            return True
+        # Check all clusters are deleted
+        list_gen = self.cloud.cluster.clusters()
+        return next(list_gen, None) is None
+
+    def list(self):
+        if not self.cloud.has_service('clustering'):
+            return []
+        return self.cloud.cluster.profiles()
+
+    def delete(self, resource):
+        self.cloud.cluster.delete_profile(profile=resource['id'])
+
+    @staticmethod
+    def to_str(resource):
+        return "Profile (id='{}', name='{}')".format(
+            resource['id'], resource['name'])

ospurge/resources/senlin.pyi

@@ -0,0 +1,69 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+from ospurge.resources import base
+
+
+class Receivers(base.ServiceResource):
+    def list(self) -> Iterable:
+        ...
+
+    def delete(self, resource: Dict[str, Any]) -> None:
+        ...
+
+    @staticmethod
+    def to_str(resource: Dict[str, Any]) -> str:
+        ...
+
+class Policies(base.ServiceResource):
+    def list(self) -> Iterable:
+        ...
+
+    def delete(self, resource: Dict[str, Any]) -> None:
+        ...
+
+    def disable(self, resource: Dict[str, Any]) -> None:
+        ...
+
+    @staticmethod
+    def to_str(resource: Dict[str, Any]) -> str:
+        ...
+
+class Clusters(base.ServiceResource):
+    def check_prerequisite(self) -> bool:
+        ...
+
+    def list(self) -> Iterable:
+        ...
+
+    def delete(self, resource: Dict[str, Any]) -> None:
+        ...
+
+    def disable(self, resource: Dict[str, Any]) -> None:
+        ...
+
+    @staticmethod
+    def to_str(resource: Dict[str, Any]) -> str:
+        ...
+
+class Profiles(base.ServiceResource):
+    def check_prerequisite(self) -> bool:
+        ...
+
+    def list(self) -> Iterable:
+        ...
+
+    def delete(self, resource: Dict[str, Any]) -> None:
+        ...
+
+    @staticmethod
+    def to_str(resource: Dict[str, Any]) -> str:
+        ...

ospurge/tests/resources/test_senlin.py

@@ -0,0 +1,258 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+import unittest
+
+import openstack.connection
+
+from ospurge.resources import senlin
+from ospurge.tests import mock
+from unittest.mock import patch
+
+
+class TestReceivers(unittest.TestCase):
+    def setUp(self):
+        self.cloud = mock.Mock(spec_set=openstack.connection.Connection)
+        self.creds_manager = mock.Mock(cloud=self.cloud)
+
+    def test_list_with_service(self):
+        self.cloud.has_service.return_value = True
+        self.assertEqual(
+            self.cloud.cluster.
+            receivers.return_value,
+            senlin.Receivers(self.creds_manager).list()
+        )
+        self.cloud.has_service.assert_called_once_with('clustering')
+        self.cloud.cluster.receivers.assert_called_once_with()
+
+    def test_list_without_service(self):
+        self.cloud.has_service.return_value = False
+        self.assertEqual(
+            [], senlin.Receivers(self.creds_manager).list()
+        )
+        self.cloud.has_service.assert_called_once_with('clustering')
+        self.cloud.cluster.receivers.assert_not_called()
+
+    def test_delete(self):
+        receiver = mock.MagicMock()
+        self.assertIsNone(senlin.Receivers(self.creds_manager).
+                          delete(receiver))
+        self.cloud.cluster.delete_receiver.\
+            assert_called_once_with(
+                receiver['id']
+            )
+
+    def test_to_string(self):
+        receiver = mock.MagicMock()
+        self.assertIn("Receiver (",
+                      senlin.Receivers(self.creds_manager).
+                      to_str(receiver))
+
+
+class TestPolicies(unittest.TestCase):
+    def setUp(self):
+        self.cloud = mock.Mock(spec_set=openstack.connection.Connection)
+        self.cluster = mock.MagicMock()
+        self.policy = mock.MagicMock()
+        self.cloud.cluster.clusters.return_value = [self.cluster]
+        self.cloud.cluster.cluster_policies.return_value = [self.policy]
+        self.creds_manager = mock.Mock(cloud=self.cloud)
+        self.ATTACHED_CLUSTERS = {self.policy['id']: [self.cluster.id]}
+        self.policy_obj = senlin.Policies(self.creds_manager)
+        self.policy_obj.ATTACHED_CLUSTERS = self.ATTACHED_CLUSTERS
+
+    def test_populate_bindings(self):
+        senlin.Policies(self.creds_manager).populate_bindings()
+        self.cloud.cluster.clusters.assert_called_once_with()
+        self.cloud.cluster.cluster_policies.assert_called_once_with(
+            self.cluster.id
+        )
+
+    def test_populate_bindings_second_call(self):
+        self.policy_obj.populate_bindings()
+        self.cloud.cluster.clusters.assert_not_called()
+        self.cloud.cluster.cluster_policies.assert_not_called()
+
+    @patch.object(senlin.Policies, 'populate_bindings')
+    def test_list_with_service(self, mock_binding):
+        self.cloud.has_service.return_value = True
+        self.assertEqual(
+            self.cloud.cluster.
+            policies.return_value,
+            senlin.Policies(self.creds_manager).list()
+        )
+        mock_binding.assert_called_once_with()
+        self.cloud.has_service.assert_called_once_with('clustering')
+        self.cloud.cluster.policies.assert_called_once_with()
+
+    @patch.object(senlin.Policies, 'populate_bindings')
+    def test_list_without_service(self, mock_binding):
+        self.cloud.has_service.return_value = False
+        self.assertEqual(
+            [], senlin.Policies(self.creds_manager).list()
+        )
+        self.cloud.has_service.assert_called_once_with('clustering')
+        mock_binding.assert_not_called()
+        self.cloud.cluster.policies.assert_not_called()
+
+    @patch.object(senlin.Policies, 'populate_bindings')
+    def test_delete(self, mock_binding):
+        self.assertIsNone(self.policy_obj.delete(self.policy))
+        self.cloud.detach_policy_from_cluster.\
+            assert_called_once_with(
+                policy=self.policy['id'],
+                cluster=self.cluster.id,
+                wait=True
+            )
+        self.cloud.cluster.delete_policy.assert_called_once_with(
+            policy=self.policy['id'])
+
+    def test_disable(self):
+        self.assertIsNone(self.policy_obj.disable(self.policy))
+        self.cloud.cluster.update_cluster_policy.\
+            assert_called_once_with(
+                policy=self.policy['id'],
+                cluster=self.cluster.id,
+                enabled=False
+            )
+
+    def test_to_string(self):
+        policy = mock.MagicMock()
+        self.assertIn("Policy (",
+                      senlin.Policies(self.creds_manager).
+                      to_str(policy))
+
+
+class TestClusters(unittest.TestCase):
+    def setUp(self):
+        self.cloud = mock.Mock(spec_set=openstack.connection.Connection)
+        self.creds_manager = mock.Mock(cloud=self.cloud)
+        self.spec = {
+            'min_size': 0,
+            'max_size': 0,
+            'adjustment_type': 'EXACT_CAPACITY',
+            'number': 0
+        }
+
+    def test_check_prerequisite_with_service(self):
+        self.cloud.has_service.return_value = True
+        self.cloud.cluster.policies.return_value = mock.MagicMock()
+        senlin.Clusters(self.creds_manager).check_prerequisite()
+        self.cloud.cluster.policies.assert_called_once_with()
+        self.cloud.cluster.receivers.assert_called_once_with()
+        self.cloud.has_service.assert_called_once_with('clustering')
+
+    def test_check_prerequisite_without_service(self):
+        self.cloud.has_service.return_value = False
+        senlin.Clusters(self.creds_manager).check_prerequisite()
+        self.cloud.cluster.policies.assert_not_called()
+        self.cloud.cluster.receivers.assert_not_called()
+        self.cloud.has_service.assert_called_once_with('clustering')
+
+    def test_list_with_service(self):
+        self.cloud.has_service.return_value = True
+        self.assertEqual(
+            self.cloud.cluster.
+            clusters.return_value,
+            senlin.Clusters(self.creds_manager).list()
+        )
+        self.cloud.has_service.assert_called_once_with('clustering')
+        self.cloud.cluster.clusters.assert_called_once_with()
+
+    def test_list_without_service(self):
+        self.cloud.has_service.return_value = False
+        self.assertEqual(
+            [], senlin.Clusters(self.creds_manager).list()
+        )
+        self.cloud.has_service.assert_called_once_with('clustering')
+        self.cloud.cluster.clusters.assert_not_called()
+
+    def test_delete(self):
+        cluster = mock.MagicMock()
+        self.assertIsNone(senlin.Clusters(self.creds_manager).
+                          delete(cluster))
+        self.cloud.cluster.delete_cluster.\
+            assert_called_once_with(
+                cluster=cluster['id']
+            )
+
+    def test_disable(self):
+        cluster = mock.MagicMock()
+        self.assertIsNone(senlin.Clusters(self.creds_manager).
+                          disable(cluster))
+        self.cloud.cluster.resize_cluster.\
+            assert_called_once_with(
+                cluster=cluster['id'],
+                **self.spec
+            )
+
+    def test_to_string(self):
+        cluster = mock.MagicMock()
+        self.assertIn("Cluster (",
+                      senlin.Clusters(self.creds_manager).
+                      to_str(cluster))
+
+
+class TestProfiles(unittest.TestCase):
+    def setUp(self):
+        self.cloud = mock.Mock(spec_set=openstack.connection.Connection)
+        self.creds_manager = mock.Mock(cloud=self.cloud)
+
+    def test_check_prerequisite_with_service(self):
+        self.cloud.has_service.return_value = True
+        self.cloud.cluster.clusters.return_value = mock.MagicMock()
+        senlin.Profiles(self.creds_manager).check_prerequisite()
+        self.cloud.cluster.clusters.assert_called_once_with()
+        self.cloud.has_service.assert_called_once_with('clustering')
+
+    def test_check_prerequisite_without_service(self):
+        self.cloud.has_service.return_value = False
+        senlin.Profiles(self.creds_manager).check_prerequisite()
+        self.cloud.cluster.clusters.assert_not_called()
+        self.cloud.has_service.assert_called_once_with('clustering')
+
+    def test_list_with_service(self):
+        self.cloud.has_service.return_value = True
+        self.assertEqual(
+            self.cloud.cluster.
+            profiles.return_value,
+            senlin.Profiles(self.creds_manager).list()
+        )
+        self.cloud.has_service.assert_called_once_with('clustering')
+        self.cloud.cluster.profiles.assert_called_once_with()
+
+    def test_list_without_service(self):
+        self.cloud.has_service.return_value = False
+        self.assertEqual(
+            [], senlin.Profiles(self.creds_manager).list()
+        )
+        self.cloud.has_service.assert_called_once_with('clustering')
+        self.cloud.cluster.profiles.assert_not_called()
+
+    def test_delete(self):
+        profile = mock.MagicMock()
+        self.assertIsNone(senlin.Profiles(self.creds_manager).
+                          delete(profile))
+        self.cloud.cluster.delete_profile.\
+            assert_called_once_with(
+                profile=profile['id']
+            )
+
+    def test_disable(self):
+        profile = mock.MagicMock()
+        with self.assertLogs(level='WARNING'):
+            senlin.Profiles(self.creds_manager).disable(profile)
+
+    def test_to_string(self):
+        profile = mock.MagicMock()
+        self.assertIn("Profile (",
+                      senlin.Profiles(self.creds_manager).
+                      to_str(profile))

test-requirements.txt

@@ -8,6 +8,7 @@ testrepository>=0.0.18  # Apache-2.0/BSD
 python-openstackclient>=3.16.1
 python-designateclient
 python-octaviaclient
+python-senlinclient
 
 # Python 2.7 dependencies
 mock; python_version < '3.0'

tools/func-tests.sh

@@ -147,6 +147,18 @@ function test_swift_disable {
     done
 }
 
+function test_senlin_disable {
+    for cluster in $(openstack cluster list -c id -f value); do
+        if [[ $(openstack cluster policy binding list $cluster -c is_enabled -f value | grep -ic 'true') -gt 0 ]]; then
+            echo "Some of the policies is not disabled yet :)"
+            exit 1
+        fi
+        if [[ $(openstack cluster show $cluster -c desired_capacity -f value) -ne 0 ]]; then
+            echo "Some of the clusters is not disabled yet :)"
+            exit 1
+        fi
+    done
+}
 
 ########################
 ### Pre check
@@ -198,8 +210,8 @@ assert_compute && assert_network && assert_volume
 
 tox -e run -- --os-cloud devstack --purge-own-project --verbose --disable-only # disable demo/demo
 test_neutron_disable && test_cinder_disable && test_glance_disable \
-&& test_nova_disable && test_loadbalancer_disable && test_swift_disable
+&& test_nova_disable && test_loadbalancer_disable && test_swift_disable \
-
+&& test_senlin_disable
 ########################
 ### Cleanup
 ########################
@@ -281,3 +293,23 @@ if [[ $(openstack loadbalancer list | wc -l) -ne 1 ]]; then
     echo "Not all loadbalancers were cleaned up"
     exit 1
 fi
+
+if [[ $(openstack cluster list | wc -l) -ne 1 ]]; then
+    echo "Not all clusters were cleaned up"
+    exit 1
+fi
+
+if [[ $(openstack cluster profile list | wc -l) -ne 1 ]]; then
+    echo "Not all profiles were cleaned up"
+    exit 1
+fi
+
+if [[ $(openstack cluster policy list | wc -l) -ne 1 ]]; then
+    echo "Not all policies were cleaned up"
+    exit 1
+fi
+
+if [[ $(openstack cluster receiver list | wc -l) -ne 1 ]]; then
+    echo "Not all Receivers were cleaned up"
+    exit 1
+fi

tools/populate.sh

@@ -90,6 +90,11 @@ VMIMG_NAME=${VMIMG_NAME:-cirros-0.4.0-x86_64-disk}
 ZONE_NAME="${ASCII_UUID//-/}.com."
 # LoadBalancer name used for the Octavia LoadBalancer
 LB_NAME="lb-${UUID//-/}"
+# For senlin
+CL_NAME="cl-${ASCII_UUID//-/}"
+PR_NAME="pr-${ASCII_UUID//-/}"
+PL_NAME="pl-${ASCII_UUID//-/}"
+RC_NAME="rc-${ASCII_UUID//-/}"
 LB_LISTENER_NAME="listener-${UUID//-/}"
 # Subnet used for the Octavia LoadBalancer VIP
 LB_VIP_SUBNET_ID=${LB_VIP_SUBNET_ID:-$UUID}
@@ -214,6 +219,28 @@ openstack zone create --email hostmaster@example.com ${ZONE_NAME}
 exit_on_failure "Unable to create Designate Zone ${ZONE_NAME}"
 
 
+###############################
+### Senlin
+###############################
+# Create a cluster
+IMAGE_NAME=$(openstack image list | awk "/ $VMIMG_NAME /{print \$4}")
+profile_spec="/tmp/profile_spec.yaml"
+policy_spec="/tmp/policy_spec.yaml"
+echo -e "type: senlin.policy.deletion\nversion: 1.0\ndescription: test " > $policy_spec
+echo -e "properties:\n  criteria: OLDEST_FIRST\n  destroy_after_deletion: True" >> $policy_spec
+echo -e "type: os.nova.server\nversion: 1.0\nproperties:\n  name: clustering-test" > $profile_spec
+echo -e "  flavor: $FLAVOR\n  image: "$IMAGE_NAME"\n  networks:\n   - network: $EXTNET_NAME" >> $profile_spec
+profile_status=$(openstack cluster profile create --spec-file $profile_spec $PR_NAME)
+exit_on_failure "Unable to create profile (${profile_status}) as $OS_USERNAME/$OS_PROJECT_NAME"
+policy_status=$(openstack cluster policy create --spec-file $policy_spec $PL_NAME)
+exit_on_failure "Unable to create policy (${policy_status}) as $OS_USERNAME/$OS_PROJECT_NAME"
+cluster_status=$(openstack cluster create --desired-capacity 1 --min-size 0 --max-size 1 --profile $PR_NAME $CL_NAME)
+exit_on_failure "Unable to create cluster (${cluster_status}) as $OS_USERNAME/$OS_PROJECT_NAME"
+attach_policy=$(openstack cluster policy attach --policy $PL_NAME $CL_NAME)
+exit_on_failure "Unable to attach policy to cluster (${attach_policy}) as $OS_USERNAME/$OS_PROJECT_NAME"
+attach_receiver=$(openstack cluster receiver create --cluster $CL_NAME --action CLUSTER_SCALE_OUT --type webhook $RC_NAME)
+exit_on_failure "Unable to attach receiver to cluster (${attach_receiver}) as $OS_USERNAME/$OS_PROJECT_NAME"
+
 ###############################
 ### Octavia
 ###############################