Necessary extending of objects to perform upgrade
The upgrade procedure should create a new cluster with the same parameters as the original cluster. The new method was intoduced to parameters the cluster that was created with. The return value of the new method is compatible with the data that are expected by the create method. A set of hooks of the extension mechanism was extended by the fire_callback_on_cluster_delete hook. This hooks is fired when cluster is deleted. Two opposite method were added: get_assigned_vips and assign_given_vips_for_net_groups. The first method returns all assigned VIPs of the cluster and the second complementary method receives the same format that the first returns to assign given VIPs. The patch has some impact on the get_admin_network_group method. This impact consist of adding a new filter called node_group_id that allows to filter network groups by ID of the node group when there are no any nodes in the cluster. Implements blueprint: nailgun-api-env-upgrade-extensions Change-Id: I156d3c71a7e832a2eb99a1058a6ac628fab71483
This commit is contained in:
parent
b3972d61b5
commit
48b48ba902
@ -23,3 +23,4 @@ from nailgun.extensions.base import fire_callback_on_node_collection_delete
|
|||||||
from nailgun.extensions.base import fire_callback_on_node_create
|
from nailgun.extensions.base import fire_callback_on_node_create
|
||||||
from nailgun.extensions.base import fire_callback_on_node_update
|
from nailgun.extensions.base import fire_callback_on_node_update
|
||||||
from nailgun.extensions.base import fire_callback_on_node_reset
|
from nailgun.extensions.base import fire_callback_on_node_reset
|
||||||
|
from nailgun.extensions.base import fire_callback_on_cluster_delete
|
||||||
|
@ -99,6 +99,11 @@ def fire_callback_on_node_collection_delete(node_ids):
|
|||||||
extension.on_node_collection_delete(node_ids)
|
extension.on_node_collection_delete(node_ids)
|
||||||
|
|
||||||
|
|
||||||
|
def fire_callback_on_cluster_delete(cluster):
|
||||||
|
for extension in get_all_extensions():
|
||||||
|
extension.on_cluster_delete(cluster)
|
||||||
|
|
||||||
|
|
||||||
@six.add_metaclass(abc.ABCMeta)
|
@six.add_metaclass(abc.ABCMeta)
|
||||||
class BaseExtension(object):
|
class BaseExtension(object):
|
||||||
|
|
||||||
@ -172,3 +177,8 @@ class BaseExtension(object):
|
|||||||
def on_node_collection_delete(cls, node_ids):
|
def on_node_collection_delete(cls, node_ids):
|
||||||
"""Callback which gets executed when node collection is deleted
|
"""Callback which gets executed when node collection is deleted
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def on_cluster_delete(cls, cluster):
|
||||||
|
"""Callback which gets executed when cluster is deleted
|
||||||
|
"""
|
||||||
|
@ -337,6 +337,70 @@ class NetworkManager(object):
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_assigned_vips_for_net_groups(cls, cluster):
|
||||||
|
node_group_id = objects.Cluster.get_controllers_group_id(cluster)
|
||||||
|
cluster_vips = db.query(IPAddr).join(IPAddr.network_data).filter(
|
||||||
|
IPAddr.node.is_(None) &
|
||||||
|
IPAddr.vip_type.isnot(None) &
|
||||||
|
(NetworkGroup.group_id == node_group_id)
|
||||||
|
)
|
||||||
|
return cluster_vips
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_assigned_vips(cls, cluster):
|
||||||
|
"""Return assigned VIPs mapped to names of network groups.
|
||||||
|
|
||||||
|
:param cluster: Is an instance of :class:`objects.Cluster`.
|
||||||
|
:returns: A dict of VIPs mapped to names of network groups and
|
||||||
|
they are grouped by the type.
|
||||||
|
"""
|
||||||
|
cluster_vips = cls._get_assigned_vips_for_net_groups(cluster)
|
||||||
|
vips = defaultdict(dict)
|
||||||
|
for vip in cluster_vips:
|
||||||
|
vips[vip.network_data.name][vip.vip_type] = vip.ip_addr
|
||||||
|
return vips
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def assign_given_vips_for_net_groups(cls, cluster, vips):
|
||||||
|
"""Assign given VIP addresses for network groups.
|
||||||
|
|
||||||
|
This method is the opposite of the :func:`get_assigned_vips_ips`
|
||||||
|
and compatible with results it returns. The method primarily
|
||||||
|
used for the upgrading procedure of clusters to copy VIPs from
|
||||||
|
one cluster to the other.
|
||||||
|
|
||||||
|
:param cluster: Is an instance of :class:`objects.Cluster`.
|
||||||
|
:param vips: A dict of VIPs mapped to names of network groups
|
||||||
|
that are grouped by the type.
|
||||||
|
:raises: errors.AssignIPError
|
||||||
|
"""
|
||||||
|
cluster_vips = cls._get_assigned_vips_for_net_groups(cluster)
|
||||||
|
assigned_vips = defaultdict(dict)
|
||||||
|
for vip in cluster_vips:
|
||||||
|
assigned_vips[vip.network_data.name][vip.vip_type] = vip
|
||||||
|
for net_group in cluster.network_groups:
|
||||||
|
if net_group.name not in vips:
|
||||||
|
continue
|
||||||
|
assigned_vips_by_type = assigned_vips.get(net_group.name, {})
|
||||||
|
for vip_type, ip_addr in six.iteritems(vips[net_group.name]):
|
||||||
|
if not cls.check_ip_belongs_to_net(ip_addr, net_group):
|
||||||
|
raise errors.AssignIPError(
|
||||||
|
"Cannot assign VIP with the address \"{0}\" because "
|
||||||
|
"it does not belong to the network \"{1}\""
|
||||||
|
.format(ip_addr, net_group.name))
|
||||||
|
if vip_type in assigned_vips_by_type:
|
||||||
|
assigned_vip = assigned_vips_by_type[vip_type]
|
||||||
|
assigned_vip.ip_addr = ip_addr
|
||||||
|
else:
|
||||||
|
vip = IPAddr(
|
||||||
|
network=net_group.id,
|
||||||
|
ip_addr=ip_addr,
|
||||||
|
vip_type=vip_type,
|
||||||
|
)
|
||||||
|
db().add(vip)
|
||||||
|
db().flush()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def assign_vips_for_net_groups_for_api(cls, cluster):
|
def assign_vips_for_net_groups_for_api(cls, cluster):
|
||||||
return cls.assign_vips_for_net_groups(cluster)
|
return cls.assign_vips_for_net_groups(cluster)
|
||||||
@ -1201,6 +1265,14 @@ class NetworkManager(object):
|
|||||||
elif cluster.net_provider == 'nova_network':
|
elif cluster.net_provider == 'nova_network':
|
||||||
cls.create_nova_network_config(cluster)
|
cls.create_nova_network_config(cluster)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_network_config_create_data(cls, cluster):
|
||||||
|
data = {}
|
||||||
|
if cluster.net_provider == consts.CLUSTER_NET_PROVIDERS.neutron:
|
||||||
|
data['net_l23_provider'] = cluster.network_config.net_l23_provider
|
||||||
|
data['net_segment_type'] = cluster.network_config.segmentation_type
|
||||||
|
return data
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_default_gateway(cls, node_id):
|
def get_default_gateway(cls, node_id):
|
||||||
"""Returns GW from Admin network if it's set, else returns Admin IP.
|
"""Returns GW from Admin network if it's set, else returns Admin IP.
|
||||||
|
@ -30,6 +30,7 @@ from nailgun import consts
|
|||||||
from nailgun.db import db
|
from nailgun.db import db
|
||||||
from nailgun.db.sqlalchemy import models
|
from nailgun.db.sqlalchemy import models
|
||||||
from nailgun.errors import errors
|
from nailgun.errors import errors
|
||||||
|
from nailgun.extensions import fire_callback_on_cluster_delete
|
||||||
from nailgun.extensions import fire_callback_on_node_collection_delete
|
from nailgun.extensions import fire_callback_on_node_collection_delete
|
||||||
from nailgun.logger import logger
|
from nailgun.logger import logger
|
||||||
from nailgun.objects import NailgunCollection
|
from nailgun.objects import NailgunCollection
|
||||||
@ -244,6 +245,7 @@ class Cluster(NailgunObject):
|
|||||||
_id for (_id,) in
|
_id for (_id,) in
|
||||||
db().query(models.Node.id).filter_by(cluster_id=instance.id)]
|
db().query(models.Node.id).filter_by(cluster_id=instance.id)]
|
||||||
fire_callback_on_node_collection_delete(node_ids)
|
fire_callback_on_node_collection_delete(node_ids)
|
||||||
|
fire_callback_on_cluster_delete(instance)
|
||||||
super(Cluster, cls).delete(instance)
|
super(Cluster, cls).delete(instance)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -875,6 +877,26 @@ class Cluster(NailgunObject):
|
|||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_create_data(cls, instance):
|
||||||
|
"""Return common parameters cluster was created with.
|
||||||
|
|
||||||
|
This method is compatible with :func:`create` and used to create
|
||||||
|
a new cluster with the same settings including the network
|
||||||
|
configuration.
|
||||||
|
|
||||||
|
:returns: a dict of key-value pairs as a cluster create data
|
||||||
|
"""
|
||||||
|
data = {
|
||||||
|
"name": instance.name,
|
||||||
|
"mode": instance.mode,
|
||||||
|
"net_provider": instance.net_provider,
|
||||||
|
"release_id": instance.release.id,
|
||||||
|
}
|
||||||
|
data.update(cls.get_network_manager(instance).
|
||||||
|
get_network_config_create_data(instance))
|
||||||
|
return data
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_vmware_attributes(cls, instance):
|
def get_vmware_attributes(cls, instance):
|
||||||
"""Get VmwareAttributes instance from DB. Now we have
|
"""Get VmwareAttributes instance from DB. Now we have
|
||||||
|
@ -25,6 +25,7 @@ from netaddr import IPRange
|
|||||||
from sqlalchemy import not_
|
from sqlalchemy import not_
|
||||||
|
|
||||||
import nailgun
|
import nailgun
|
||||||
|
from nailgun.errors import errors
|
||||||
from nailgun import objects
|
from nailgun import objects
|
||||||
|
|
||||||
from nailgun.db.sqlalchemy.models import IPAddr
|
from nailgun.db.sqlalchemy.models import IPAddr
|
||||||
@ -39,7 +40,27 @@ from nailgun.test.base import BaseIntegrationTest
|
|||||||
from nailgun.test.base import fake_tasks
|
from nailgun.test.base import fake_tasks
|
||||||
|
|
||||||
|
|
||||||
class TestNetworkManager(BaseIntegrationTest):
|
class BaseNetworkManagerTest(BaseIntegrationTest):
|
||||||
|
def _create_ip_addrs_by_rules(self, cluster, rules):
|
||||||
|
created_ips = []
|
||||||
|
for net_group in cluster.network_groups:
|
||||||
|
if net_group.name not in rules:
|
||||||
|
continue
|
||||||
|
vips_by_types = rules[net_group.name]
|
||||||
|
for vip_type, ip_addr in vips_by_types.items():
|
||||||
|
ip = IPAddr(
|
||||||
|
network=net_group.id,
|
||||||
|
ip_addr=ip_addr,
|
||||||
|
vip_type=vip_type,
|
||||||
|
)
|
||||||
|
self.db.add(ip)
|
||||||
|
created_ips.append(ip)
|
||||||
|
if created_ips:
|
||||||
|
self.db.flush()
|
||||||
|
return created_ips
|
||||||
|
|
||||||
|
|
||||||
|
class TestNetworkManager(BaseNetworkManagerTest):
|
||||||
|
|
||||||
@fake_tasks(fake_rpc=False, mock_rpc=False)
|
@fake_tasks(fake_rpc=False, mock_rpc=False)
|
||||||
@patch('nailgun.rpc.cast')
|
@patch('nailgun.rpc.cast')
|
||||||
@ -346,6 +367,71 @@ class TestNetworkManager(BaseIntegrationTest):
|
|||||||
self.assertEqual(len(admin_ips), 1)
|
self.assertEqual(len(admin_ips), 1)
|
||||||
self.assertEqual(admin_ips[0].ip_addr, ip)
|
self.assertEqual(admin_ips[0].ip_addr, ip)
|
||||||
|
|
||||||
|
def test_get_assigned_vips(self):
|
||||||
|
vips_to_create = {
|
||||||
|
'management': {
|
||||||
|
'haproxy': '192.168.0.1',
|
||||||
|
'vrouter': '192.168.0.2',
|
||||||
|
},
|
||||||
|
'public': {
|
||||||
|
'haproxy': '172.16.0.2',
|
||||||
|
'vrouter': '172.16.0.3',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cluster = self.env.create_cluster(api=False)
|
||||||
|
self._create_ip_addrs_by_rules(cluster, vips_to_create)
|
||||||
|
vips = self.env.network_manager.get_assigned_vips(cluster)
|
||||||
|
self.assertEqual(vips_to_create, vips)
|
||||||
|
|
||||||
|
def test_assign_given_vips_for_net_groups(self):
|
||||||
|
vips_to_create = {
|
||||||
|
'management': {
|
||||||
|
'haproxy': '192.168.0.1',
|
||||||
|
},
|
||||||
|
'public': {
|
||||||
|
'haproxy': '172.16.0.2',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
vips_to_assign = {
|
||||||
|
'management': {
|
||||||
|
'haproxy': '192.168.0.1',
|
||||||
|
'vrouter': '192.168.0.2',
|
||||||
|
},
|
||||||
|
'public': {
|
||||||
|
'haproxy': '172.16.0.4',
|
||||||
|
'vrouter': '172.16.0.5',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
cluster = self.env.create_cluster(api=False)
|
||||||
|
self._create_ip_addrs_by_rules(cluster, vips_to_create)
|
||||||
|
self.env.network_manager.assign_given_vips_for_net_groups(
|
||||||
|
cluster, vips_to_assign)
|
||||||
|
vips = self.env.network_manager.get_assigned_vips(cluster)
|
||||||
|
self.assertEqual(vips_to_assign, vips)
|
||||||
|
|
||||||
|
def test_assign_given_vips_for_net_groups_idempotent(self):
|
||||||
|
cluster = self.env.create_cluster(api=False)
|
||||||
|
self.env.network_manager.assign_vips_for_net_groups(cluster)
|
||||||
|
expected_vips = self.env.network_manager.get_assigned_vips(cluster)
|
||||||
|
self.env.network_manager.assign_given_vips_for_net_groups(
|
||||||
|
cluster, expected_vips)
|
||||||
|
self.env.network_manager.assign_vips_for_net_groups(cluster)
|
||||||
|
vips = self.env.network_manager.get_assigned_vips(cluster)
|
||||||
|
self.assertEqual(expected_vips, vips)
|
||||||
|
|
||||||
|
def test_assign_given_vips_for_net_groups_assign_error(self):
|
||||||
|
vips_to_assign = {
|
||||||
|
'management': {
|
||||||
|
'haproxy': '10.10.0.1',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
expected_msg_regexp = '^Cannot assign VIP with the address "10.10.0.1"'
|
||||||
|
cluster = self.env.create_cluster(api=False)
|
||||||
|
with self.assertRaisesRegexp(errors.AssignIPError,
|
||||||
|
expected_msg_regexp):
|
||||||
|
self.env.network_manager.assign_given_vips_for_net_groups(
|
||||||
|
cluster, vips_to_assign)
|
||||||
|
|
||||||
@fake_tasks(fake_rpc=False, mock_rpc=False)
|
@fake_tasks(fake_rpc=False, mock_rpc=False)
|
||||||
@patch('nailgun.rpc.cast')
|
@patch('nailgun.rpc.cast')
|
||||||
def test_admin_ip_cobbler(self, mocked_rpc):
|
def test_admin_ip_cobbler(self, mocked_rpc):
|
||||||
@ -511,7 +597,7 @@ class TestNeutronManager(BaseIntegrationTest):
|
|||||||
self.check_networks_assignment(self.env.nodes[0])
|
self.check_networks_assignment(self.env.nodes[0])
|
||||||
|
|
||||||
|
|
||||||
class TestNeutronManager70(BaseIntegrationTest):
|
class TestNeutronManager70(BaseNetworkManagerTest):
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestNeutronManager70, self).setUp()
|
super(TestNeutronManager70, self).setUp()
|
||||||
@ -614,3 +700,43 @@ class TestNeutronManager70(BaseIntegrationTest):
|
|||||||
self.assertEqual(assigned_vips[name]['node_roles'],
|
self.assertEqual(assigned_vips[name]['node_roles'],
|
||||||
['controller',
|
['controller',
|
||||||
'primary-controller'])
|
'primary-controller'])
|
||||||
|
|
||||||
|
def test_get_assigned_vips(self):
|
||||||
|
self.net_manager.assign_vips_for_net_groups(self.cluster)
|
||||||
|
vips = self.net_manager.get_assigned_vips(self.cluster)
|
||||||
|
expected_vips = {
|
||||||
|
'management': {
|
||||||
|
'vrouter': '192.168.0.1',
|
||||||
|
'management': '192.168.0.2',
|
||||||
|
},
|
||||||
|
'public': {
|
||||||
|
'vrouter_pub': '172.16.0.2',
|
||||||
|
'public': '172.16.0.3',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
self.assertEqual(expected_vips, vips)
|
||||||
|
|
||||||
|
def test_assign_given_vips_for_net_groups(self):
|
||||||
|
vips_to_create = {
|
||||||
|
'management': {
|
||||||
|
'vrouter': '192.168.0.1',
|
||||||
|
},
|
||||||
|
'public': {
|
||||||
|
'vrouter_pub': '172.16.0.2',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
vips_to_assign = {
|
||||||
|
'management': {
|
||||||
|
'vrouter': '192.168.0.2',
|
||||||
|
'management': '192.168.0.3',
|
||||||
|
},
|
||||||
|
'public': {
|
||||||
|
'vrouter_pub': '172.16.0.4',
|
||||||
|
'public': '172.16.0.5',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
self._create_ip_addrs_by_rules(self.cluster, vips_to_create)
|
||||||
|
self.net_manager.assign_given_vips_for_net_groups(
|
||||||
|
self.cluster, vips_to_assign)
|
||||||
|
vips = self.net_manager.get_assigned_vips(self.cluster)
|
||||||
|
self.assertEqual(vips_to_assign, vips)
|
||||||
|
@ -18,6 +18,7 @@ import mock
|
|||||||
|
|
||||||
from nailgun.errors import errors
|
from nailgun.errors import errors
|
||||||
from nailgun.extensions import BaseExtension
|
from nailgun.extensions import BaseExtension
|
||||||
|
from nailgun.extensions import fire_callback_on_cluster_delete
|
||||||
from nailgun.extensions import fire_callback_on_node_collection_delete
|
from nailgun.extensions import fire_callback_on_node_collection_delete
|
||||||
from nailgun.extensions import fire_callback_on_node_create
|
from nailgun.extensions import fire_callback_on_node_create
|
||||||
from nailgun.extensions import fire_callback_on_node_delete
|
from nailgun.extensions import fire_callback_on_node_delete
|
||||||
@ -181,3 +182,12 @@ class TestExtensionUtils(BaseTestCase):
|
|||||||
|
|
||||||
for ext in get_m.return_value:
|
for ext in get_m.return_value:
|
||||||
ext.on_node_collection_delete.assert_called_once_with(node_ids)
|
ext.on_node_collection_delete.assert_called_once_with(node_ids)
|
||||||
|
|
||||||
|
@mock.patch('nailgun.extensions.base.get_all_extensions',
|
||||||
|
return_value=make_mock_extensions())
|
||||||
|
def test_fire_callback_on_cluster_deletion(self, get_m):
|
||||||
|
cluster = mock.MagicMock()
|
||||||
|
fire_callback_on_cluster_delete(cluster)
|
||||||
|
|
||||||
|
for ext in get_m.return_value:
|
||||||
|
ext.on_cluster_delete.assert_called_once_with(cluster)
|
||||||
|
@ -536,6 +536,32 @@ class TestNodeObject(BaseIntegrationTest):
|
|||||||
self.assertEqual(node.ip_addrs, [])
|
self.assertEqual(node.ip_addrs, [])
|
||||||
self.assertEqual(node.pending_roles, prev_roles)
|
self.assertEqual(node.pending_roles, prev_roles)
|
||||||
|
|
||||||
|
def _assert_cluster_create_data(self, network_data):
|
||||||
|
release = self.env.create_release(api=False)
|
||||||
|
expected_data = {
|
||||||
|
"name": "cluster-0",
|
||||||
|
"mode": consts.CLUSTER_MODES.ha_compact,
|
||||||
|
"release_id": release.id,
|
||||||
|
}
|
||||||
|
expected_data.update(network_data)
|
||||||
|
cluster = self.env.create_cluster(api=False, **expected_data)
|
||||||
|
create_data = objects.Cluster.get_create_data(cluster)
|
||||||
|
self.assertEqual(expected_data, create_data)
|
||||||
|
|
||||||
|
def test_cluster_get_create_data_neutron(self):
|
||||||
|
network_data = {
|
||||||
|
"net_provider": consts.CLUSTER_NET_PROVIDERS.neutron,
|
||||||
|
"net_segment_type": consts.NEUTRON_SEGMENT_TYPES.vlan,
|
||||||
|
"net_l23_provider": consts.NEUTRON_L23_PROVIDERS.ovs,
|
||||||
|
}
|
||||||
|
self._assert_cluster_create_data(network_data)
|
||||||
|
|
||||||
|
def test_cluster_get_create_data_nova(self):
|
||||||
|
network_data = {
|
||||||
|
"net_provider": consts.CLUSTER_NET_PROVIDERS.nova_network,
|
||||||
|
}
|
||||||
|
self._assert_cluster_create_data(network_data)
|
||||||
|
|
||||||
|
|
||||||
class TestTaskObject(BaseIntegrationTest):
|
class TestTaskObject(BaseIntegrationTest):
|
||||||
|
|
||||||
@ -858,14 +884,16 @@ class TestClusterObject(BaseTestCase):
|
|||||||
network_role.update(kwargs)
|
network_role.update(kwargs)
|
||||||
return network_role
|
return network_role
|
||||||
|
|
||||||
|
@mock.patch('nailgun.objects.cluster.fire_callback_on_cluster_delete')
|
||||||
@mock.patch(
|
@mock.patch(
|
||||||
'nailgun.objects.cluster.'
|
'nailgun.objects.cluster.'
|
||||||
'fire_callback_on_node_collection_delete')
|
'fire_callback_on_node_collection_delete')
|
||||||
def test_delete(self, callback_mock):
|
def test_delete(self, mock_node_coll_delete_cb, mock_cluster_delete_cb):
|
||||||
cluster = self.env.clusters[0]
|
cluster = self.env.clusters[0]
|
||||||
ids = [node.id for node in cluster.nodes]
|
ids = [node.id for node in cluster.nodes]
|
||||||
objects.Cluster.delete(cluster)
|
objects.Cluster.delete(cluster)
|
||||||
callback_mock.assert_called_once_with(ids)
|
mock_node_coll_delete_cb.assert_called_once_with(ids)
|
||||||
|
mock_cluster_delete_cb.assert_called_once_with(cluster)
|
||||||
self.assertEqual(self.db.query(objects.Node.model).count(), 0)
|
self.assertEqual(self.db.query(objects.Node.model).count(), 0)
|
||||||
self.assertEqual(self.db.query(objects.Cluster.model).count(), 0)
|
self.assertEqual(self.db.query(objects.Cluster.model).count(), 0)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user