Release 0.5.0:
- added support to wstun allow list - added force renew rpc/api - added DeviceRestSubmit allowed action - added checks in create and enable webservice - added ServicesStatus RPC/API - added hard delete device procedures - added support for LR "freedom" edition - added checks if dns or webservice already exists - check in device creation for duplicated device name - updated nginx confs Change-Id: Idb95e63cc4d732f69331ebb585ee576ae872ac5c
This commit is contained in:
Nicola Peditto
parent
6e7ddfaf21
commit
3e5782eb1f
@ -416,7 +416,8 @@ class BoardPluginsController(rest.RestController):
|
||||
class BoardServicesController(rest.RestController):
|
||||
_custom_actions = {
|
||||
'action': ['POST'],
|
||||
'restore': ['GET']
|
||||
'restore': ['GET'],
|
||||
'status': ['GET']
|
||||
}
|
||||
|
||||
def __init__(self, board_ident):
|
||||
@ -462,6 +463,7 @@ class BoardServicesController(rest.RestController):
|
||||
except exception:
|
||||
return exception
|
||||
|
||||
# add here the check on force parameter
|
||||
rpc_board.check_if_online()
|
||||
|
||||
result = pecan.request.rpcapi.action_service(pecan.request.context,
|
||||
@ -491,11 +493,33 @@ class BoardServicesController(rest.RestController):
|
||||
|
||||
return self._get_services_on_board_collection(rpc_board.uuid)
|
||||
|
||||
@expose.expose(wtypes.text, status_code=200)
|
||||
def status(self):
|
||||
"""Get status of all services in a board.
|
||||
|
||||
:param board_ident: UUID or logical name of a board.
|
||||
"""
|
||||
# context = pecan.request.context
|
||||
# cdict = context.to_policy_values()
|
||||
# policy.authorize('iot:board:delete', cdict, cdict)
|
||||
|
||||
rpc_board = api_utils.get_rpc_board(self.board_ident)
|
||||
|
||||
rpc_board.check_if_online()
|
||||
|
||||
result = pecan.request.rpcapi.status_services_on_board(
|
||||
pecan.request.context,
|
||||
rpc_board.uuid
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class BoardWebservicesController(rest.RestController):
|
||||
_custom_actions = {
|
||||
'enable': ['POST'],
|
||||
'disable': ['DELETE']
|
||||
'disable': ['DELETE'],
|
||||
'renew': ['GET']
|
||||
}
|
||||
|
||||
invalid_sort_key_list = ['extra', ]
|
||||
@ -575,6 +599,7 @@ class BoardWebservicesController(rest.RestController):
|
||||
|
||||
:param Webservice: a Webservice within the request body.
|
||||
"""
|
||||
|
||||
context = pecan.request.context
|
||||
cdict = context.to_policy_values()
|
||||
policy.authorize('iot:webservice:create', cdict, cdict)
|
||||
@ -652,6 +677,23 @@ class BoardWebservicesController(rest.RestController):
|
||||
pecan.request.rpcapi.disable_webservice(pecan.request.context,
|
||||
rpc_board.uuid)
|
||||
|
||||
@expose.expose(None)
|
||||
def renew(self):
|
||||
"""Renew webservices certificates in a board.
|
||||
|
||||
:param board_ident: UUID or logical name of a board.
|
||||
"""
|
||||
# context = pecan.request.context
|
||||
# cdict = context.to_policy_values()
|
||||
# policy.authorize('iot:board:delete', cdict, cdict)
|
||||
|
||||
rpc_board = api_utils.get_rpc_board(self.board_ident)
|
||||
|
||||
rpc_board.check_if_online()
|
||||
|
||||
pecan.request.rpcapi.renew_webservice(pecan.request.context,
|
||||
rpc_board.uuid)
|
||||
|
||||
|
||||
class BoardPortsController(rest.RestController):
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ class Location(base.APIBase):
|
||||
fields = list(objects.Location.fields)
|
||||
for k in fields:
|
||||
# Skip fields we do not expose.
|
||||
if k != 'created_at':
|
||||
if k is not 'created_at':
|
||||
if not hasattr(self, k):
|
||||
continue
|
||||
self.fields.append(k)
|
||||
|
||||
@ -327,7 +327,6 @@ class LocalLinkConnectionType(wtypes.UserType):
|
||||
return None
|
||||
return LocalLinkConnectionType.validate(value)
|
||||
|
||||
|
||||
locallinkconnectiontype = LocalLinkConnectionType()
|
||||
|
||||
|
||||
|
||||
@ -260,6 +260,10 @@ class BoardAssociated(InvalidState):
|
||||
message = _("Board %(board)s is associated with instance %(instance)s.")
|
||||
|
||||
|
||||
class BoardNameAlreadyExists(Conflict):
|
||||
message = _("A board with %(name)s is already associated to another board.")
|
||||
|
||||
|
||||
class PortNotFound(NotFound):
|
||||
message = _("Port %(port)s could not be found.")
|
||||
|
||||
@ -665,8 +669,12 @@ class WebserviceAlreadyExists(Conflict):
|
||||
|
||||
|
||||
class EnabledWebserviceNotFound(NotFound):
|
||||
message = _("No Webservice enabled for %(enabled_webservice)s.")
|
||||
message = _("Webservice module not enabled for %(enabled_webservice)s device.")
|
||||
|
||||
|
||||
class EnabledWebserviceAlreadyExists(Conflict):
|
||||
message = _("Already enabled for %(enabled_webservice)s.")
|
||||
message = _("Already enabled for %(enabled_webservice)s [req: %(req)s].")
|
||||
|
||||
|
||||
class DnsWebserviceAlreadyExists(Conflict):
|
||||
message = _("DNS %(dns)s already exists [req: %(req)s].")
|
||||
@ -23,6 +23,7 @@ except Exception:
|
||||
|
||||
import random
|
||||
import socket
|
||||
import json
|
||||
|
||||
|
||||
from oslo_config import cfg
|
||||
@ -46,6 +47,7 @@ Port = list()
|
||||
SERVICE_PORT_LIST = []
|
||||
|
||||
|
||||
|
||||
def versionCompare(v1, v2):
|
||||
"""Method to compare two versions.
|
||||
Return 1 if v2 is smaller,
|
||||
@ -53,6 +55,11 @@ def versionCompare(v1, v2):
|
||||
0 if equal
|
||||
"""
|
||||
|
||||
if "freedom" in v1:
|
||||
return 1
|
||||
|
||||
else:
|
||||
|
||||
v1_list = v1.split(".")[:3]
|
||||
v2_list = v2.split(".")[:3]
|
||||
i = 0
|
||||
@ -154,6 +161,7 @@ def create_record_dns_webservice(ctx, board, webs_name, board_dns, zone):
|
||||
LOG.debug('using %s %s %s', webs_name + "." + board_dns,
|
||||
ip, zone)
|
||||
|
||||
# CHECK IF DNS EXISTS
|
||||
designate.create_record(webs_name + "." + board_dns, ip,
|
||||
zone)
|
||||
|
||||
@ -170,8 +178,8 @@ def create_record_dns(ctx, board, board_dns, zone):
|
||||
LOG.debug('using %s %s %s', board_dns,
|
||||
ip, zone)
|
||||
|
||||
designate.create_record(board_dns, ip,
|
||||
zone)
|
||||
# CHECK IF DNS EXISTS
|
||||
designate.create_record(board_dns, ip, zone)
|
||||
|
||||
LOG.debug('Configure Web Proxy on WampAgent %s (%s) for board %s',
|
||||
board.agent, ip, board.uuid)
|
||||
@ -253,20 +261,34 @@ class ConductorEndpoint(object):
|
||||
board_id)
|
||||
board = objects.Board.get_by_uuid(ctx, board_id)
|
||||
result = None
|
||||
|
||||
if board.is_online():
|
||||
prov = Provisioner()
|
||||
prov.conf_clean()
|
||||
p = prov.get_config()
|
||||
LOG.debug('sending this conf %s', p)
|
||||
|
||||
try:
|
||||
result = self.execute_on_board(ctx,
|
||||
board_id,
|
||||
'destroyBoard',
|
||||
(p,))
|
||||
|
||||
'DeviceFactoryReset',
|
||||
())
|
||||
except exception:
|
||||
return exception
|
||||
|
||||
exposed_list = objects.ExposedService.get_by_board_uuid(ctx,
|
||||
board_id)
|
||||
|
||||
LOG.debug('starting the wamp client')
|
||||
cctx = self.wamp_agent_client.prepare(server=board.agent)
|
||||
|
||||
LOG.debug('Service to remove from allowlist:')
|
||||
LOG.debug(exposed_list)
|
||||
|
||||
for exposed in exposed_list:
|
||||
LOG.debug(exposed.public_port)
|
||||
cctx.call(ctx, 'remove_from_allowlist',
|
||||
device=board_id,
|
||||
port=exposed.public_port)
|
||||
|
||||
board.destroy()
|
||||
|
||||
if result:
|
||||
result = manage_result(result, 'destroyBoard', board_id)
|
||||
LOG.debug(result)
|
||||
@ -280,9 +302,16 @@ class ConductorEndpoint(object):
|
||||
return serializer.serialize_entity(ctx, board)
|
||||
|
||||
def create_board(self, ctx, board_obj, location_obj):
|
||||
|
||||
new_board = serializer.deserialize_entity(ctx, board_obj)
|
||||
LOG.debug('Creating board %s',
|
||||
new_board.name)
|
||||
LOG.debug('Creating board %s', new_board.name)
|
||||
|
||||
board_exists = objects.Board.exists_by_name(ctx, new_board.name)
|
||||
#print(board_exists)
|
||||
|
||||
if board_exists:
|
||||
raise exception.BoardNameAlreadyExists(name=new_board.name)
|
||||
else:
|
||||
new_location = serializer.deserialize_entity(ctx, location_obj)
|
||||
new_board.create()
|
||||
new_location.board_id = new_board.id
|
||||
@ -292,7 +321,7 @@ class ConductorEndpoint(object):
|
||||
|
||||
def execute_on_board(self, ctx, board_uuid, wamp_rpc_call, wamp_rpc_args,
|
||||
main_req=None):
|
||||
LOG.debug('Executing \"%s\" on the board: %s (req %s)',
|
||||
LOG.debug('Executing \"%s\" on the board: %s (main_req %s)',
|
||||
wamp_rpc_call, board_uuid, main_req)
|
||||
|
||||
board = objects.Board.get_by_uuid(ctx, board_uuid)
|
||||
@ -308,18 +337,20 @@ class ConductorEndpoint(object):
|
||||
|
||||
req = new_req(ctx, board, objects.request.BOARD, wamp_rpc_call,
|
||||
main_req)
|
||||
LOG.info(" - exec_request: " + str(req.uuid))
|
||||
|
||||
res = new_res(ctx, board, req.uuid)
|
||||
|
||||
cctx = self.wamp_agent_client.prepare(server=board.agent)
|
||||
|
||||
# for previous LR version (to be removed asap)
|
||||
|
||||
if (versionCompare(board.lr_version, "0.4.9") == -1):
|
||||
|
||||
response = cctx.call(ctx, 's4t_invoke_wamp',
|
||||
wamp_rpc_call=full_wamp_call,
|
||||
data=wamp_rpc_args)
|
||||
else:
|
||||
|
||||
response = cctx.call(ctx, 's4t_invoke_wamp',
|
||||
wamp_rpc_call=full_wamp_call,
|
||||
req=req,
|
||||
@ -484,6 +515,10 @@ class ConductorEndpoint(object):
|
||||
service = objects.Service.get(ctx, service_uuid)
|
||||
objects.service.is_valid_action(action)
|
||||
|
||||
board = objects.Board.get_by_uuid(ctx, board_uuid)
|
||||
if not board.is_online():
|
||||
raise exception.BoardNotConnected(board=board.uuid)
|
||||
|
||||
if action == "ServiceEnable":
|
||||
LOG.info('Enabling service with id %s into the board %s',
|
||||
service_uuid, board_uuid)
|
||||
@ -493,11 +528,23 @@ class ConductorEndpoint(object):
|
||||
service_uuid)
|
||||
return exception.ServiceAlreadyExposed(uuid=service_uuid)
|
||||
except Exception:
|
||||
|
||||
public_port = random_public_port()
|
||||
|
||||
if not public_port:
|
||||
return exception.NotEnoughPortForService()
|
||||
|
||||
# Add into service allow list
|
||||
#if cfg.CONF.conductor.service_allow_list:
|
||||
# addin_allowlist(board_uuid, public_port)
|
||||
LOG.debug('starting the wamp client')
|
||||
cctx = self.wamp_agent_client.prepare(server=board.agent)
|
||||
cctx.call(ctx, 'addin_allowlist',
|
||||
device=board_uuid,
|
||||
port=public_port)
|
||||
|
||||
|
||||
|
||||
res = self.execute_on_board(ctx, board_uuid, action,
|
||||
(service, public_port))
|
||||
result = manage_result(res, action, board_uuid)
|
||||
@ -531,6 +578,18 @@ class ConductorEndpoint(object):
|
||||
except exception:
|
||||
pass
|
||||
exposed.destroy()
|
||||
|
||||
|
||||
# Remove from service allow list
|
||||
#if cfg.CONF.conductor.service_allow_list:
|
||||
#remove_from_allowlist(board_uuid, exposed.public_port)
|
||||
LOG.debug('starting the wamp client')
|
||||
cctx = self.wamp_agent_client.prepare(server=board.agent)
|
||||
cctx.call(ctx, 'remove_from_allowlist',
|
||||
device=board_uuid,
|
||||
port=exposed.public_port)
|
||||
|
||||
|
||||
return result
|
||||
|
||||
elif action == "ServiceRestore":
|
||||
@ -559,6 +618,25 @@ class ConductorEndpoint(object):
|
||||
|
||||
return 0
|
||||
|
||||
def status_services_on_board(self, ctx, board_uuid):
|
||||
LOG.info('Getting services status into devide %s',
|
||||
board_uuid)
|
||||
|
||||
action = "ServicesStatus"
|
||||
|
||||
board = objects.Board.get(ctx, board_uuid)
|
||||
if not board.is_online():
|
||||
raise exception.BoardNotConnected(board=board.uuid)
|
||||
|
||||
try:
|
||||
result = self.execute_on_board(ctx, board_uuid, action, ())
|
||||
except exception:
|
||||
return exception
|
||||
|
||||
result = manage_result(result, action, board_uuid)
|
||||
LOG.debug(result)
|
||||
return result
|
||||
|
||||
def create_port_on_board(self, ctx, board_uuid, network_uuid,
|
||||
subnet_uuid, security_groups=None):
|
||||
|
||||
@ -691,13 +769,33 @@ class ConductorEndpoint(object):
|
||||
|
||||
def create_webservice(self, ctx, webservice_obj):
|
||||
newwbs = serializer.deserialize_entity(ctx, webservice_obj)
|
||||
LOG.debug('Creating webservice %s',
|
||||
newwbs.name)
|
||||
LOG.debug('Creating webservice %s', newwbs.name)
|
||||
|
||||
board = objects.Board.get_by_uuid(ctx, newwbs.board_uuid)
|
||||
if not board.is_online():
|
||||
raise exception.BoardNotConnected(board=board.uuid)
|
||||
|
||||
LOG.info(" - expose "+str(newwbs.name)+" in " + str(board.name) + " ["+str(newwbs.board_uuid)+"]")
|
||||
list_webs = objects.Webservice.list(
|
||||
ctx,
|
||||
filters={'board_uuid': newwbs.board_uuid}
|
||||
)
|
||||
|
||||
dns_check=True
|
||||
webs_found=None
|
||||
|
||||
for webs in list_webs:
|
||||
wboard = objects.Board.get(ctx, webs.board_uuid)
|
||||
LOG.info(" --> " + str(webs.name) + ", " + str(wboard.name) + ", " + str(webs.board_uuid) + ", " + str(webs.port) )
|
||||
if (str(newwbs.board_uuid) == str(webs.board_uuid)) and (newwbs.name == str(webs.name)):
|
||||
dns_check = False
|
||||
webs_found = webs
|
||||
break
|
||||
|
||||
if dns_check:
|
||||
|
||||
LOG.info("Webservice can be exposed!")
|
||||
|
||||
en_webservice = objects.enabledwebservice. \
|
||||
EnabledWebservice.get_by_board_uuid(ctx,
|
||||
newwbs.board_uuid)
|
||||
@ -712,10 +810,6 @@ class ConductorEndpoint(object):
|
||||
LOG.debug('Creating webservice with full domain %s',
|
||||
dns_domain)
|
||||
|
||||
list_webs = objects.Webservice.list(ctx,
|
||||
filters={
|
||||
'board_uuid': newwbs.board_uuid
|
||||
})
|
||||
list_dns = full_zone_domain + ","
|
||||
for webs in list_webs:
|
||||
dname = webs.name + "." + full_zone_domain + ","
|
||||
@ -724,7 +818,7 @@ class ConductorEndpoint(object):
|
||||
list_dns = list_dns + dns_domain
|
||||
|
||||
try:
|
||||
self.execute_on_board(ctx,
|
||||
res = self.execute_on_board(ctx,
|
||||
newwbs.board_uuid,
|
||||
'ExposeWebservice',
|
||||
(full_zone_domain,
|
||||
@ -741,8 +835,37 @@ class ConductorEndpoint(object):
|
||||
|
||||
newwbs.create()
|
||||
|
||||
result = manage_result(res, "ExposeWebservice", board.uuid)
|
||||
LOG.info(result)
|
||||
#return result
|
||||
|
||||
return serializer.serialize_entity(ctx, newwbs)
|
||||
|
||||
else:
|
||||
|
||||
mreq = new_req(ctx, board, objects.request.BOARD,
|
||||
"expose_webservice", pending_requests=0)
|
||||
LOG.info(" - request: " + str(mreq.uuid))
|
||||
|
||||
|
||||
|
||||
msg="Webservice already exposed for this device!"
|
||||
w_msg = wm.WampWarning(msg, req_id=mreq.uuid)
|
||||
|
||||
res = new_res(ctx, board, mreq.uuid)
|
||||
res.result = w_msg.result
|
||||
res.message = w_msg.message
|
||||
res.save()
|
||||
|
||||
mreq.status = objects.request.COMPLETED
|
||||
mreq.save()
|
||||
|
||||
result = manage_result(w_msg, "ExposeWebservice", board.uuid)
|
||||
|
||||
LOG.info(result)
|
||||
#return result
|
||||
return serializer.serialize_entity(ctx, webs_found)
|
||||
|
||||
def destroy_webservice(self, ctx, webservice_id):
|
||||
LOG.info('Destroying webservice with id %s',
|
||||
webservice_id)
|
||||
@ -750,13 +873,16 @@ class ConductorEndpoint(object):
|
||||
wbsrv = objects.Webservice.get_by_uuid(ctx, webservice_id)
|
||||
|
||||
board = objects.Board.get_by_uuid(ctx, wbsrv.board_uuid)
|
||||
if not board.is_online():
|
||||
raise exception.BoardNotConnected(board=board.uuid)
|
||||
|
||||
#if not board.is_online():
|
||||
# raise exception.BoardNotConnected(board=board.uuid)
|
||||
|
||||
en_webservice = objects.enabledwebservice. \
|
||||
EnabledWebservice.get_by_board_uuid(ctx,
|
||||
wbsrv.board_uuid)
|
||||
|
||||
if board.is_online():
|
||||
|
||||
full_zone_domain = en_webservice.dns + "." + en_webservice.zone
|
||||
dns_domain = wbsrv.name + "." + full_zone_domain
|
||||
|
||||
@ -793,6 +919,7 @@ class ConductorEndpoint(object):
|
||||
wbsrv.destroy()
|
||||
designate.delete_record(wbsrv.name + "." + en_webservice.dns,
|
||||
en_webservice.zone)
|
||||
|
||||
return
|
||||
|
||||
def enable_webservice(self, ctx, dns, zone, email, board_uuid):
|
||||
@ -804,6 +931,52 @@ class ConductorEndpoint(object):
|
||||
if board.agent == None:
|
||||
raise exception.BoardInvalidStatus(uuid=board.uuid,
|
||||
status=board.status)
|
||||
#print("DNS: " + str(dns))
|
||||
|
||||
avl_webservice = objects.enabledwebservice.EnabledWebservice.checkDnsAvailable(ctx, dns)
|
||||
#print("Available: " + str(avl_webservice))
|
||||
|
||||
LOG.info("Enabling Webservice module for device " + str(board.name) + " ["+board_uuid+"]")
|
||||
|
||||
if avl_webservice:
|
||||
|
||||
en_webservice = objects.enabledwebservice.EnabledWebservice.isWebserviceEnabled(ctx, board.uuid)
|
||||
|
||||
if(en_webservice):
|
||||
|
||||
mreq = new_req(ctx, board, objects.request.BOARD,
|
||||
"enable_webservice", pending_requests=0)
|
||||
LOG.info(" - request: " + str(mreq.uuid))
|
||||
|
||||
res = new_res(ctx, board, mreq.uuid)
|
||||
|
||||
en_webservice = objects.enabledwebservice.EnabledWebservice.get_by_board_uuid(ctx, board.uuid)
|
||||
|
||||
LOG.info("Webservice and dns already enabled for this device:")
|
||||
full_zone_domain = en_webservice.dns + "." + en_webservice.zone
|
||||
|
||||
LOG.info(" --> " + str(full_zone_domain))
|
||||
|
||||
msg="Webservice and dns already enabled for this device!"
|
||||
w_msg = wm.WampWarning(msg, req_id=mreq.uuid)
|
||||
|
||||
res.result = w_msg.result
|
||||
res.message = w_msg.message
|
||||
res.save()
|
||||
|
||||
mreq.status = objects.request.COMPLETED
|
||||
mreq.save()
|
||||
|
||||
result = manage_result(w_msg, "EnableWebservice", board.uuid)
|
||||
|
||||
LOG.info(result)
|
||||
|
||||
raise exception.EnabledWebserviceAlreadyExists(enabled_webservice=full_zone_domain, req=mreq.uuid)
|
||||
|
||||
#return serializer.serialize_entity(ctx, en_webservice)
|
||||
|
||||
|
||||
else:
|
||||
|
||||
mreq = new_req(ctx, board, objects.request.BOARD,
|
||||
"enable_webservice", pending_requests=3)
|
||||
@ -849,6 +1022,16 @@ class ConductorEndpoint(object):
|
||||
|
||||
service = objects.Service.get_by_name(ctx, 'webservice')
|
||||
|
||||
|
||||
LOG.debug('starting the wamp client')
|
||||
cctx = self.wamp_agent_client.prepare(server=board.agent)
|
||||
cctx.call(ctx, 'addin_allowlist',
|
||||
device=board.uuid,
|
||||
port=http_port)
|
||||
cctx.call(ctx, 'addin_allowlist',
|
||||
device=board.uuid,
|
||||
port=https_port)
|
||||
|
||||
res = self.execute_on_board(ctx, board.uuid, "ServiceEnable",
|
||||
(service, http_port,), main_req=mreq.uuid)
|
||||
result = manage_result(res, "ServiceEnable", board.uuid)
|
||||
@ -865,6 +1048,10 @@ class ConductorEndpoint(object):
|
||||
|
||||
service = objects.Service.get_by_name(ctx, 'webservice_ssl')
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
res = self.execute_on_board(ctx, board.uuid, "ServiceEnable",
|
||||
(service, https_port,), main_req=mreq.uuid)
|
||||
result = manage_result(res, "ServiceEnable", board.uuid)
|
||||
@ -901,13 +1088,38 @@ class ConductorEndpoint(object):
|
||||
|
||||
return serializer.serialize_entity(ctx, en_webservice)
|
||||
|
||||
|
||||
else:
|
||||
mreq = new_req(ctx, board, objects.request.BOARD,
|
||||
"enable_webservice", pending_requests=0)
|
||||
LOG.info(" - request: " + str(mreq.uuid))
|
||||
|
||||
res = new_res(ctx, board, mreq.uuid)
|
||||
|
||||
msg="DNS already exists!"
|
||||
w_msg = wm.WampWarning(msg, req_id=mreq.uuid)
|
||||
|
||||
res.result = w_msg.result
|
||||
res.message = w_msg.message
|
||||
res.save()
|
||||
|
||||
mreq.status = objects.request.COMPLETED
|
||||
mreq.save()
|
||||
|
||||
result = manage_result(w_msg, "EnableWebservice", board.uuid)
|
||||
|
||||
LOG.info(" - result: " + str(result))
|
||||
|
||||
raise exception.DnsWebserviceAlreadyExists(dns=dns, req=mreq.uuid)
|
||||
|
||||
def disable_webservice(self, ctx, board_uuid):
|
||||
LOG.info('Disabling webservice on board id %s',
|
||||
board_uuid)
|
||||
|
||||
board = objects.Board.get_by_uuid(ctx, board_uuid)
|
||||
if not board.is_online():
|
||||
raise exception.BoardNotConnected(board=board.uuid)
|
||||
|
||||
#if not board.is_online():
|
||||
# raise exception.BoardNotConnected(board=board.uuid)
|
||||
|
||||
if board.agent == None:
|
||||
raise exception.BoardInvalidStatus(uuid=board.uuid,
|
||||
@ -932,9 +1144,11 @@ class ConductorEndpoint(object):
|
||||
board_uuid,
|
||||
service.uuid)
|
||||
|
||||
if board.is_online():
|
||||
res = self.execute_on_board(ctx, board.uuid, "ServiceDisable",
|
||||
(service,), main_req=mreq.uuid)
|
||||
LOG.debug(res.message)
|
||||
|
||||
exposed.destroy()
|
||||
|
||||
service = objects.Service.get_by_name(ctx, 'webservice_ssl')
|
||||
@ -942,9 +1156,13 @@ class ConductorEndpoint(object):
|
||||
exposed = objects.ExposedService.get(ctx,
|
||||
board_uuid,
|
||||
service.uuid)
|
||||
|
||||
if board.is_online():
|
||||
res = self.execute_on_board(ctx, board.uuid, "ServiceDisable",
|
||||
(service,), main_req=mreq.uuid)
|
||||
LOG.debug(res.message)
|
||||
|
||||
|
||||
global SERVICE_PORT_LIST
|
||||
try:
|
||||
SERVICE_PORT_LIST.append(exposed.public_port)
|
||||
@ -952,6 +1170,7 @@ class ConductorEndpoint(object):
|
||||
pass
|
||||
exposed.destroy()
|
||||
|
||||
if board.is_online():
|
||||
try:
|
||||
self.execute_on_board(ctx,
|
||||
board.uuid,
|
||||
@ -969,6 +1188,7 @@ class ConductorEndpoint(object):
|
||||
|
||||
designate.delete_record(webservice.dns, en_webservice.zone)
|
||||
|
||||
if board.is_online():
|
||||
cctx = self.wamp_agent_client.prepare(server=board.agent)
|
||||
cctx.call(ctx, 'disable_webservice', board=webservice.dns)
|
||||
|
||||
@ -977,5 +1197,64 @@ class ConductorEndpoint(object):
|
||||
|
||||
cctx.call(ctx, 'reload_proxy')
|
||||
|
||||
|
||||
LOG.debug('starting the wamp client')
|
||||
cctx = self.wamp_agent_client.prepare(server=board.agent)
|
||||
cctx.call(ctx, 'remove_from_allowlist',
|
||||
device=board.uuid,
|
||||
port=http_port)
|
||||
|
||||
|
||||
cctx.call(ctx, 'remove_from_allowlist',
|
||||
device=board.uuid,
|
||||
port=https_port)
|
||||
|
||||
webservice.destroy()
|
||||
|
||||
return
|
||||
|
||||
def renew_webservice(self, ctx, board_uuid):
|
||||
|
||||
board = objects.Board.get_by_uuid(ctx, board_uuid)
|
||||
if not board.is_online():
|
||||
raise exception.BoardNotConnected(board=board.uuid)
|
||||
|
||||
if board.agent == None:
|
||||
raise exception.BoardInvalidStatus(uuid=board.uuid,
|
||||
status=board.status)
|
||||
|
||||
LOG.info('Renewing certificates on device %s', board.uuid)
|
||||
|
||||
en_webservice = objects.enabledwebservice.EnabledWebservice.isWebserviceEnabled(ctx, board.uuid)
|
||||
|
||||
if(en_webservice):
|
||||
|
||||
mreq = new_req(ctx, board, objects.request.BOARD,
|
||||
"RenewWebservice", pending_requests=1)
|
||||
|
||||
res = self.execute_on_board(ctx, board.uuid, "RenewWebservice", (), main_req=mreq.uuid )
|
||||
|
||||
else:
|
||||
|
||||
mreq = new_req(ctx, board, objects.request.BOARD,
|
||||
"RenewWebservice", pending_requests=0)
|
||||
LOG.info(" - request: " + str(mreq.uuid))
|
||||
|
||||
msg="Webservice module not enabled for this device!"
|
||||
w_msg = wm.WampWarning(msg, req_id=mreq.uuid)
|
||||
|
||||
res = new_res(ctx, board, mreq.uuid)
|
||||
res.result = w_msg.result
|
||||
res.message = w_msg.message
|
||||
res.save()
|
||||
|
||||
mreq.status = objects.request.COMPLETED
|
||||
mreq.save()
|
||||
|
||||
raise exception.EnabledWebserviceNotFound(enabled_webservice=board.uuid)
|
||||
|
||||
|
||||
result = manage_result(res, "RenewWebservice", board.uuid)
|
||||
LOG.info(" - " + str(result))
|
||||
|
||||
return serializer.serialize_entity(ctx, en_webservice)
|
||||
|
||||
@ -46,6 +46,7 @@ conductor_opts = [
|
||||
cfg.IntOpt('service_port_max',
|
||||
default=60000,
|
||||
help='Max value for genereting random ports for services'),
|
||||
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
@ -291,6 +291,20 @@ class ConductorAPI(object):
|
||||
return cctxt.call(context, 'restore_services_on_board',
|
||||
board_uuid=board_uuid)
|
||||
|
||||
|
||||
def status_services_on_board(self, context,
|
||||
board_uuid, topic=None):
|
||||
"""Get status of all services' tunnel on a board.
|
||||
|
||||
:param context: request context.
|
||||
:param board_uuid: board id or uuid.
|
||||
|
||||
"""
|
||||
cctxt = self.client.prepare(topic=topic or self.topic, version='1.0')
|
||||
return cctxt.call(context, 'status_services_on_board',
|
||||
board_uuid=board_uuid)
|
||||
|
||||
|
||||
def create_port_on_board(self, context, board_uuid, network,
|
||||
subnet, sec_groups, topic=None):
|
||||
"""Add a port on a Board
|
||||
@ -409,3 +423,11 @@ class ConductorAPI(object):
|
||||
cctxt = self.client.prepare(topic=topic or self.topic, version='1.0')
|
||||
return cctxt.call(context, 'disable_webservice',
|
||||
board_uuid=board_uuid)
|
||||
|
||||
def renew_webservice(self, context, board_uuid, topic=None):
|
||||
"""Renew webservice certificate.
|
||||
|
||||
"""
|
||||
cctxt = self.client.prepare(topic=topic or self.topic, version='1.0')
|
||||
return cctxt.call(context, 'renew_webservice',
|
||||
board_uuid=board_uuid)
|
||||
|
||||
@ -0,0 +1,23 @@
|
||||
"""extended board code attribute
|
||||
|
||||
Revision ID: 10460765f337
|
||||
Revises: 57a99337e843
|
||||
Create Date: 2021-11-03 16:17:19.310762
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '10460765f337'
|
||||
down_revision = '57a99337e843'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
def upgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.alter_column('boards', 'code',
|
||||
existing_type=mysql.VARCHAR(length=36),
|
||||
nullable=False)
|
||||
|
||||
# ### end Alembic commands ###
|
||||
@ -0,0 +1,64 @@
|
||||
"""hard device delete
|
||||
|
||||
Revision ID: 57a99337e843
|
||||
Revises: 76c628d60004
|
||||
Create Date: 2020-07-07 15:45:20.892424
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '57a99337e843'
|
||||
down_revision = '76c628d60004'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
def upgrade():
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_constraint('enabled_webservices_ibfk_1', 'enabled_webservices', type_='foreignkey')
|
||||
op.create_foreign_key(None, 'enabled_webservices', 'boards', ['board_uuid'], ['uuid'], ondelete='CASCADE')
|
||||
op.drop_constraint('exposed_services_ibfk_1', 'exposed_services', type_='foreignkey')
|
||||
op.create_foreign_key(None, 'exposed_services', 'boards', ['board_uuid'], ['uuid'], ondelete='CASCADE')
|
||||
op.drop_constraint('injection_plugins_ibfk_1', 'injection_plugins', type_='foreignkey')
|
||||
op.create_foreign_key(None, 'injection_plugins', 'boards', ['board_uuid'], ['uuid'], ondelete='CASCADE')
|
||||
op.drop_constraint('locations_ibfk_1', 'locations', type_='foreignkey')
|
||||
op.create_foreign_key(None, 'locations', 'boards', ['board_id'], ['id'], ondelete='CASCADE')
|
||||
op.drop_constraint('ports_on_boards_ibfk_1', 'ports_on_boards', type_='foreignkey')
|
||||
op.create_foreign_key(None, 'ports_on_boards', 'boards', ['board_uuid'], ['uuid'], ondelete='CASCADE')
|
||||
op.alter_column('requests', 'action',
|
||||
existing_type=mysql.VARCHAR(length=20),
|
||||
nullable=True)
|
||||
op.alter_column('requests', 'destination_uuid',
|
||||
existing_type=mysql.VARCHAR(length=36),
|
||||
nullable=True)
|
||||
op.alter_column('requests', 'pending_requests',
|
||||
existing_type=mysql.INTEGER(display_width=11),
|
||||
nullable=True)
|
||||
op.alter_column('requests', 'status',
|
||||
existing_type=mysql.VARCHAR(length=10),
|
||||
nullable=True)
|
||||
op.alter_column('requests', 'type',
|
||||
existing_type=mysql.INTEGER(display_width=11),
|
||||
nullable=True)
|
||||
op.alter_column('requests', 'uuid',
|
||||
existing_type=mysql.VARCHAR(length=36),
|
||||
nullable=True)
|
||||
op.alter_column('results', 'board_uuid',
|
||||
existing_type=mysql.VARCHAR(length=36),
|
||||
nullable=True)
|
||||
op.alter_column('results', 'request_uuid',
|
||||
existing_type=mysql.VARCHAR(length=36),
|
||||
nullable=True)
|
||||
op.alter_column('results', 'result',
|
||||
existing_type=mysql.VARCHAR(length=10),
|
||||
nullable=True)
|
||||
op.drop_constraint('results_ibfk_1', 'results', type_='foreignkey')
|
||||
op.drop_constraint('results_ibfk_2', 'results', type_='foreignkey')
|
||||
op.drop_constraint('sessions_ibfk_1', 'sessions', type_='foreignkey')
|
||||
op.create_foreign_key(None, 'sessions', 'boards', ['board_id'], ['id'], ondelete='CASCADE')
|
||||
op.create_unique_constraint('uniq_webservices0uuid', 'webservices', ['uuid'])
|
||||
op.drop_index('uniq_enabled_webservices0uuid', table_name='webservices')
|
||||
op.drop_constraint('webservices_ibfk_1', 'webservices', type_='foreignkey')
|
||||
op.create_foreign_key(None, 'webservices', 'boards', ['board_uuid'], ['uuid'], ondelete='CASCADE')
|
||||
# ### end Alembic commands ###
|
||||
@ -1175,6 +1175,15 @@ class Connection(api.Connection):
|
||||
|
||||
# ENABLED_WEBSERIVCE api
|
||||
|
||||
def get_enabled_webservice_by_name(self, enabled_webservice_name):
|
||||
query = model_query(models.EnabledWebservice).filter_by(
|
||||
dns=enabled_webservice_name)
|
||||
try:
|
||||
return query.one()
|
||||
except NoResultFound:
|
||||
return False
|
||||
#exception.EnabledWebserviceNotFound(enabled_webservice=enabled_webservice_name)
|
||||
|
||||
def get_enabled_webservice_by_id(self, enabled_webservice_id):
|
||||
query = model_query(models.EnabledWebservice).filter_by(
|
||||
id=enabled_webservice_id)
|
||||
|
||||
@ -144,7 +144,7 @@ class Board(Base):
|
||||
table_args())
|
||||
id = Column(Integer, primary_key=True)
|
||||
uuid = Column(String(36))
|
||||
code = Column(String(25))
|
||||
code = Column(String(36))
|
||||
status = Column(String(15), nullable=True)
|
||||
name = Column(String(255), nullable=True)
|
||||
type = Column(String(255))
|
||||
@ -169,7 +169,7 @@ class Location(Base):
|
||||
longitude = Column(String(18), nullable=True)
|
||||
latitude = Column(String(18), nullable=True)
|
||||
altitude = Column(String(18), nullable=True)
|
||||
board_id = Column(Integer, ForeignKey('boards.id'))
|
||||
board_id = Column(Integer, ForeignKey('boards.id', ondelete="CASCADE"))
|
||||
|
||||
|
||||
class SessionWP(Base):
|
||||
@ -185,7 +185,7 @@ class SessionWP(Base):
|
||||
valid = Column(Boolean, default=True)
|
||||
session_id = Column(String(20))
|
||||
board_uuid = Column(String(36))
|
||||
board_id = Column(Integer, ForeignKey('boards.id'))
|
||||
board_id = Column(Integer, ForeignKey('boards.id', ondelete="CASCADE"))
|
||||
|
||||
|
||||
class Plugin(Base):
|
||||
@ -213,7 +213,7 @@ class InjectionPlugin(Base):
|
||||
__table_args__ = (
|
||||
table_args())
|
||||
id = Column(Integer, primary_key=True)
|
||||
board_uuid = Column(String(36), ForeignKey('boards.uuid'))
|
||||
board_uuid = Column(String(36), ForeignKey('boards.uuid', ondelete="CASCADE"))
|
||||
plugin_uuid = Column(String(36), ForeignKey('plugins.uuid'))
|
||||
onboot = Column(Boolean, default=False)
|
||||
status = Column(String(15))
|
||||
@ -242,7 +242,7 @@ class ExposedService(Base):
|
||||
__table_args__ = (
|
||||
table_args())
|
||||
id = Column(Integer, primary_key=True)
|
||||
board_uuid = Column(String(36), ForeignKey('boards.uuid'))
|
||||
board_uuid = Column(String(36), ForeignKey('boards.uuid', ondelete="CASCADE"))
|
||||
service_uuid = Column(String(36), ForeignKey('services.uuid'))
|
||||
public_port = Column(Integer)
|
||||
|
||||
@ -256,7 +256,7 @@ class Port(Base):
|
||||
# table_args()
|
||||
# )
|
||||
id = Column(Integer, primary_key=True)
|
||||
board_uuid = Column(String(40), ForeignKey('boards.uuid'))
|
||||
board_uuid = Column(String(40), ForeignKey('boards.uuid', ondelete="CASCADE"))
|
||||
uuid = Column(String(40))
|
||||
VIF_name = Column(String(30))
|
||||
# project = Column(String(36))
|
||||
@ -298,7 +298,7 @@ class Webservice(Base):
|
||||
uuid = Column(String(36))
|
||||
port = Column(Integer)
|
||||
name = Column(String(45))
|
||||
board_uuid = Column(String(36), ForeignKey('boards.uuid'), nullable=True)
|
||||
board_uuid = Column(String(36), ForeignKey('boards.uuid', ondelete="CASCADE"), nullable=True)
|
||||
secure = Column(Boolean, default=True)
|
||||
extra = Column(JSONEncodedDict)
|
||||
|
||||
@ -308,7 +308,7 @@ class EnabledWebservice(Base):
|
||||
|
||||
__tablename__ = 'enabled_webservices'
|
||||
id = Column(Integer, primary_key=True)
|
||||
board_uuid = Column(String(36), ForeignKey('boards.uuid'), nullable=True)
|
||||
board_uuid = Column(String(36), ForeignKey('boards.uuid', ondelete="CASCADE"), nullable=True)
|
||||
http_port = Column(Integer)
|
||||
https_port = Column(Integer)
|
||||
dns = Column(String(100))
|
||||
|
||||
@ -24,7 +24,7 @@ from iotronic.objects import utils as obj_utils
|
||||
|
||||
ACTIONS = ['DevicePing', 'DeviceReboot', 'DeviceRestartLR',
|
||||
'DeviceNetConfig', 'DeviceEcho', 'DeviceUpgradeLR',
|
||||
'DevicePkgOperation']
|
||||
'DevicePkgOperation', 'DeviceRestSubmit']
|
||||
|
||||
|
||||
def is_valid_action(action):
|
||||
@ -132,6 +132,22 @@ class Board(base.IotronicObject):
|
||||
board = Board._from_db_object(cls(context), db_board)
|
||||
return board
|
||||
|
||||
@base.remotable_classmethod
|
||||
def exists_by_name(cls, context, name):
|
||||
"""Find a board based on name and return a boolean.
|
||||
|
||||
:param name: the logical name of a board.
|
||||
:returns: a boolean object.
|
||||
"""
|
||||
res=False
|
||||
try:
|
||||
db_board = cls.dbapi.get_board_by_name(name)
|
||||
res = True
|
||||
except exception.BoardNotFound as E:
|
||||
return res
|
||||
|
||||
return res
|
||||
|
||||
@base.remotable_classmethod
|
||||
def list(cls, context, limit=None, marker=None, sort_key=None,
|
||||
sort_dir=None, filters=None):
|
||||
|
||||
@ -88,6 +88,46 @@ class EnabledWebservice(base.IotronicObject):
|
||||
db_enabled_webservice)
|
||||
return en_webserv
|
||||
|
||||
@base.remotable_classmethod
|
||||
def isWebserviceEnabled(cls, context, uuid):
|
||||
"""Find a enabled_webservice based on uuid and return a Board object.
|
||||
|
||||
:param uuid: the uuid of a enabled_webservice.
|
||||
:returns: a boolean object.
|
||||
"""
|
||||
res=False
|
||||
try:
|
||||
db_enabled_webservice = cls.dbapi.get_enabled_webservice_by_board_uuid(
|
||||
uuid)
|
||||
res = True
|
||||
except exception.EnabledWebserviceNotFound as E:
|
||||
return res
|
||||
|
||||
return res
|
||||
|
||||
@base.remotable_classmethod
|
||||
def checkDnsAvailable(cls, context, dns):
|
||||
"""Check if a dns is already assigned.
|
||||
|
||||
:param dns: dns chosen to enabled_webservice into a device.
|
||||
:returns: a boolean object.
|
||||
"""
|
||||
res=False
|
||||
try:
|
||||
db_enabled_webservice = cls.dbapi.get_enabled_webservice_by_name(
|
||||
dns)
|
||||
if db_enabled_webservice == False:
|
||||
# OK dns approved
|
||||
res = True
|
||||
else:
|
||||
res = False
|
||||
except exception.EnabledWebserviceAlreadyExists as E:
|
||||
return res
|
||||
|
||||
return res
|
||||
|
||||
|
||||
|
||||
@base.remotable_classmethod
|
||||
def list(cls, context, limit=None, marker=None, sort_key=None,
|
||||
sort_dir=None, filters=None):
|
||||
|
||||
@ -204,6 +204,7 @@ class Plugin(base.IotronicObject):
|
||||
"""
|
||||
current = self.__class__.get_by_uuid(self._context, self.uuid)
|
||||
for field in self.fields:
|
||||
if (hasattr(self, base.get_attrname(field))
|
||||
if (hasattr(
|
||||
self, base.get_attrname(field))
|
||||
and self[field] != current[field]):
|
||||
self[field] = current[field]
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import asyncio
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import txaio
|
||||
@ -72,6 +73,12 @@ wamp_opts = [
|
||||
cfg.IntOpt('autoPingTimeout',
|
||||
default=2,
|
||||
help=('autoPingInterval parameter for wamp')),
|
||||
cfg.BoolOpt('service_allow_list',
|
||||
default=False,
|
||||
help='Enable service allow list checks.'),
|
||||
cfg.StrOpt('service_allow_list_path',
|
||||
default="(/var/lib/wstun/allowlist)",
|
||||
help='Path of allowlist.json file.'),
|
||||
|
||||
]
|
||||
|
||||
@ -121,6 +128,20 @@ class WampEndpoint(object):
|
||||
|
||||
return r.result()
|
||||
|
||||
def read_allowlist():
|
||||
try:
|
||||
|
||||
with open(CONF.wamp.service_allow_list_path, "r") as allow_file:
|
||||
|
||||
allow_list_str = allow_file.read()
|
||||
|
||||
allow_list = json.loads(allow_list_str)
|
||||
#LOG.debug(allow_list)
|
||||
|
||||
return allow_list
|
||||
|
||||
except Exception as err:
|
||||
LOG.error(err)
|
||||
|
||||
class AgentEndpoint(object):
|
||||
|
||||
@ -140,6 +161,54 @@ class AgentEndpoint(object):
|
||||
return 1
|
||||
|
||||
|
||||
|
||||
|
||||
def addin_allowlist(self, ctx, device, port):
|
||||
try:
|
||||
|
||||
allow_list = read_allowlist()
|
||||
|
||||
new_node={}
|
||||
new_node['client']=device
|
||||
new_node['port']=str(port)
|
||||
|
||||
if new_node in allow_list:
|
||||
LOG.warning("This device already exposes this port!")
|
||||
else:
|
||||
allow_list.append(new_node)
|
||||
with open(CONF.wamp.service_allow_list_path, "r+") as allow_file:
|
||||
allow_file.seek(0)
|
||||
allow_file.write("%s" % json.dumps(allow_list))
|
||||
allow_file.truncate()
|
||||
|
||||
read_allowlist()
|
||||
LOG.debug("Added device/service port in allow list.")
|
||||
|
||||
except Exception as err:
|
||||
print(err)
|
||||
|
||||
|
||||
def remove_from_allowlist(self, ctx, device, port):
|
||||
try:
|
||||
allow_list = read_allowlist()
|
||||
|
||||
new_node={}
|
||||
new_node['client']=device
|
||||
new_node['port']=str(port)
|
||||
|
||||
if new_node in allow_list:
|
||||
allow_list.remove(new_node)
|
||||
with open(CONF.wamp.service_allow_list_path, "r+") as allow_file:
|
||||
allow_file.seek(0)
|
||||
allow_file.write("%s" % json.dumps(allow_list))
|
||||
allow_file.truncate()
|
||||
|
||||
LOG.debug("Removed device/service port from allow list.")
|
||||
|
||||
except Exception as err:
|
||||
print(err)
|
||||
|
||||
|
||||
class RPCServer(Thread):
|
||||
def __init__(self):
|
||||
# AMQP CONFIG
|
||||
|
||||
@ -23,6 +23,9 @@ LOG = logging.getLogger(__name__)
|
||||
nginx_opts = [
|
||||
cfg.StrOpt('nginx_path',
|
||||
default='/etc/nginx/conf.d/iotronic',
|
||||
help=('Default Nginx Path')),
|
||||
cfg.StrOpt('wstun_endpoint',
|
||||
default='localhost',
|
||||
help=('Default Nginx Path'))
|
||||
]
|
||||
|
||||
@ -39,9 +42,9 @@ def save_map(board, zone):
|
||||
def save_upstream(board, https_port):
|
||||
fp = CONF.nginx.nginx_path + "/upstreams/upstream_" + board
|
||||
string = '''upstream {0} {{
|
||||
server localhost:{1} max_fails=3 fail_timeout=10s;
|
||||
server {2}:{1} max_fails=3 fail_timeout=10s;
|
||||
}}
|
||||
'''.format(board, https_port)
|
||||
'''.format(board, https_port, CONF.nginx.wstun_endpoint )
|
||||
|
||||
with open(fp, "w") as text_file:
|
||||
text_file.write("%s" % string)
|
||||
@ -54,10 +57,10 @@ def save_server(board, http_port, zone):
|
||||
server_name .{0}.{2};
|
||||
|
||||
location / {{
|
||||
proxy_pass http://localhost:{1};
|
||||
proxy_pass http://{3}:{1};
|
||||
}}
|
||||
}}
|
||||
'''.format(board, http_port, zone)
|
||||
'''.format(board, http_port, zone, CONF.nginx.wstun_endpoint)
|
||||
|
||||
with open(fp, "w") as text_file:
|
||||
text_file.write("%s" % string)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user