Use san mapping queue to run background jobs
Change-Id: I720318674a7ab943e2d28980c875ec6b197672f1
This commit is contained in:
Isaac Mungai
parent
bf8d2a4c94
commit
5d150053aa
|
|
@ -28,8 +28,8 @@ class BackgroundJobControllerBase(controller.ManagerControllerBase):
|
|||
super(BackgroundJobControllerBase, self).__init__(manager)
|
||||
|
||||
@abc.abstractmethod
|
||||
def post_job(self, job_type, args):
|
||||
"""Returns the health of storage and providers
|
||||
def post_job(self, job_type, kwargs):
|
||||
"""Posts a background job to the distributed task driver.
|
||||
|
||||
:raises: NotImplementedError
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -42,66 +42,96 @@ class BackgroundJobController(base.BackgroundJobController):
|
|||
n_driver.MAIL_NOTIFICATION_GROUP].recipients
|
||||
|
||||
def post_job(self, job_type, kwargs):
|
||||
kwargs = kwargs
|
||||
queue_data = []
|
||||
|
||||
if job_type == "akamai_check_and_update_cert_status":
|
||||
LOG.info('Starting to check status on domain: %s,'
|
||||
'for project_id: %s'
|
||||
'flavor_id: %s, cert_type: %s' %
|
||||
(
|
||||
kwargs.get("domain_name"),
|
||||
kwargs.get("project_id"),
|
||||
kwargs.get("flavor_id"),
|
||||
kwargs.get("cert_type")
|
||||
))
|
||||
self.distributed_task_controller.submit_task(
|
||||
check_cert_status_and_update_flow.
|
||||
check_cert_status_and_update_flow,
|
||||
**kwargs)
|
||||
# this task will consume the san mapping queue
|
||||
if 'akamai' in self._driver.providers:
|
||||
akamai_driver = self._driver.providers['akamai'].obj
|
||||
queue_data += akamai_driver.san_mapping_queue.traverse_queue(
|
||||
consume=True
|
||||
)
|
||||
for cert_dict in queue_data:
|
||||
cert_dict = json.loads(cert_dict)
|
||||
LOG.info('Starting to check status on domain: %s,'
|
||||
'for project_id: %s'
|
||||
'flavor_id: %s, cert_type: %s' %
|
||||
(
|
||||
cert_dict.get("domain_name"),
|
||||
cert_dict.get("project_id"),
|
||||
cert_dict.get("flavor_id"),
|
||||
cert_dict.get("cert_type")
|
||||
))
|
||||
t_kwargs = {
|
||||
"cert_obj_json": json.dumps(cert_dict),
|
||||
"project_id": kwargs.get("project_id")
|
||||
}
|
||||
self.distributed_task_controller.submit_task(
|
||||
check_cert_status_and_update_flow.
|
||||
check_cert_status_and_update_flow,
|
||||
**t_kwargs
|
||||
)
|
||||
elif job_type == "akamai_update_papi_property_for_mod_san":
|
||||
update_cname_host_mapping_info = kwargs.get(
|
||||
"update_cname_host_mapping_info"
|
||||
)
|
||||
# this task will leave the san mapping queue intact
|
||||
if 'akamai' in self._driver.providers:
|
||||
akamai_driver = self._driver.providers['akamai'].obj
|
||||
queue_data += akamai_driver.san_mapping_queue.traverse_queue()
|
||||
|
||||
cname_host_info_list = []
|
||||
for domain_san_cert_cname_pair in update_cname_host_mapping_info:
|
||||
LOG.info("%s: %s to %s, on property: %s" % (
|
||||
kwargs.get("action", 'add'),
|
||||
domain_san_cert_cname_pair["domain_name"],
|
||||
domain_san_cert_cname_pair["san_cert_name"],
|
||||
kwargs.get("property_spec",
|
||||
'akamai_https_san_config_numbers')
|
||||
))
|
||||
|
||||
for cert_dict in queue_data:
|
||||
cert_dict = json.loads(cert_dict)
|
||||
|
||||
domain_name = cert_dict["domain_name"]
|
||||
san_cert = (
|
||||
cert_dict["cert_details"]
|
||||
["Akamai"]["extra_info"]["san cert"]
|
||||
)
|
||||
LOG.info(
|
||||
"{0}: {1} to {2}, on property: {3}".format(
|
||||
kwargs.get("action", 'add'),
|
||||
domain_name,
|
||||
san_cert,
|
||||
kwargs.get(
|
||||
"property_spec",
|
||||
'akamai_https_san_config_numbers'
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Note(tonytan4ever): Put this check here so erroneous
|
||||
# san cert params will not pass. Support occasionally put in
|
||||
# the ending "edgekey.net"
|
||||
# (e.g: securexxx.san1.abc.com.edgekey.net), this check will
|
||||
# effectively error that out
|
||||
if domain_san_cert_cname_pair["san_cert_name"] not in \
|
||||
self.akamai_san_cert_cname_list:
|
||||
raise ValueError("Not A valid san cert cname: %s, "
|
||||
"valid san cert cnames are: %s" %
|
||||
(kwargs.get("san_cert_name"),
|
||||
self.akamai_san_cert_cname_list))
|
||||
if san_cert not in self.akamai_san_cert_cname_list:
|
||||
raise ValueError(
|
||||
"Not A valid san cert cname: {0}, "
|
||||
"valid san cert cnames are: {1}".format(
|
||||
san_cert,
|
||||
self.akamai_san_cert_cname_list
|
||||
)
|
||||
)
|
||||
cname_host_info_list.append({
|
||||
"cnameFrom": domain_san_cert_cname_pair["domain_name"],
|
||||
"cnameTo": '.'.join([domain_san_cert_cname_pair[
|
||||
"san_cert_name"],
|
||||
kwargs.get("san_cert_domain_suffix",
|
||||
self.akamai_san_cert_suffix)]),
|
||||
"cnameFrom": domain_name,
|
||||
"cnameTo": '.'.join(
|
||||
[san_cert, self.akamai_san_cert_suffix]
|
||||
),
|
||||
"cnameType": "EDGE_HOSTNAME"
|
||||
})
|
||||
|
||||
t_kwargs = {}
|
||||
|
||||
update_info_list = json.dumps([
|
||||
(kwargs.get("action", 'add'),
|
||||
cname_host_info_list)
|
||||
(
|
||||
kwargs.get("action", 'add'),
|
||||
cname_host_info_list
|
||||
)
|
||||
])
|
||||
|
||||
t_kwargs = {
|
||||
"property_spec": kwargs.get("property_spec",
|
||||
'akamai_https_san_config_numbers'),
|
||||
"property_spec": kwargs.get(
|
||||
"property_spec",
|
||||
'akamai_https_san_config_numbers'
|
||||
),
|
||||
"update_type": kwargs.get("update_type", 'hostnames'),
|
||||
"update_info_list": update_info_list,
|
||||
"notify_email_list": self.notify_email_list
|
||||
|
|
|
|||
|
|
@ -31,7 +31,6 @@ conf(project='poppy', prog='poppy', args=[])
|
|||
|
||||
def check_cert_status_and_update_flow():
|
||||
flow = linear_flow.Flow('Update Akamai Property').add(
|
||||
check_cert_status_and_update_tasks.GetCertInfoTask(),
|
||||
check_cert_status_and_update_tasks.CheckCertStatusTask(),
|
||||
check_cert_status_and_update_tasks.UpdateCertStatusTask()
|
||||
)
|
||||
|
|
|
|||
|
|
@ -55,8 +55,8 @@ class CheckCertStatusTask(task.Task):
|
|||
|
||||
def execute(self, cert_obj_json):
|
||||
if cert_obj_json != "":
|
||||
cert_obj = ssl_certificate.load_from_json(json.loads(cert_obj_json)
|
||||
)
|
||||
cert_obj = ssl_certificate.load_from_json(
|
||||
json.loads(cert_obj_json))
|
||||
latest_sps_id = cert_obj.cert_details['Akamai']['extra_info'].get(
|
||||
'akamai_spsId')
|
||||
current_status = cert_obj.cert_details['Akamai']['extra_info'].get(
|
||||
|
|
@ -96,8 +96,15 @@ class CheckCertStatusTask(task.Task):
|
|||
elif status == 'CPS cancelled':
|
||||
return "cancelled"
|
||||
else:
|
||||
LOG.info("SPS Not completed for %s ..." %
|
||||
cert_obj.get_san_edge_name())
|
||||
LOG.info(
|
||||
"SPS Not completed for {0}. "
|
||||
"Returning certificate object to Queue.".format(
|
||||
cert_obj.get_san_edge_name()
|
||||
)
|
||||
)
|
||||
self.akamai_driver.san_mapping_queue.enqueue_san_mapping(
|
||||
cert_obj_json
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ class SanMappingQueue(object):
|
|||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def traverse_queue(self):
|
||||
def traverse_queue(self, consume=False):
|
||||
"""Traverse queue and return all items on the queue in a list"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
|
|||
|
|
@ -68,13 +68,14 @@ class ZookeeperSanMappingQueue(base.SanMappingQueue):
|
|||
def enqueue_san_mapping(self, san_domain_map):
|
||||
self.san_mapping_queue_backend.put(san_domain_map)
|
||||
|
||||
def traverse_queue(self):
|
||||
def traverse_queue(self, consume=False):
|
||||
res = []
|
||||
while len(self.san_mapping_queue_backend) > 0:
|
||||
item = self.san_mapping_queue_backend.get()
|
||||
self.san_mapping_queue_backend.consume()
|
||||
res.append(item)
|
||||
self.san_mapping_queue_backend.put_all(res)
|
||||
if consume is False:
|
||||
self.san_mapping_queue_backend.put_all(res)
|
||||
return res
|
||||
|
||||
def put_queue_data(self, queue_data):
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from poppy.transport.validators import schema_base
|
|||
|
||||
class BackgroundJobSchema(schema_base.SchemaBase):
|
||||
|
||||
'''JSON Schema validation for /admin/provider/akamai/background_jobs'''
|
||||
"""JSON Schema validation for /admin/provider/akamai/background_jobs"""
|
||||
|
||||
schema = {
|
||||
'background_jobs': {
|
||||
|
|
@ -31,22 +31,9 @@ class BackgroundJobSchema(schema_base.SchemaBase):
|
|||
'required': True,
|
||||
'enum': ['akamai_check_and_update_cert_status']
|
||||
},
|
||||
'domain_name': {
|
||||
'type': 'string',
|
||||
'required': True
|
||||
},
|
||||
'project_id': {
|
||||
'type': 'string',
|
||||
'required': True
|
||||
},
|
||||
'cert_type': {
|
||||
'type': 'string',
|
||||
'required': True,
|
||||
'enum': ['san']
|
||||
},
|
||||
'flavor_id': {
|
||||
'type': 'string',
|
||||
'required': True
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -58,23 +45,9 @@ class BackgroundJobSchema(schema_base.SchemaBase):
|
|||
'required': True,
|
||||
'enum': ['akamai_update_papi_property_for_mod_san']
|
||||
},
|
||||
'update_cname_host_mapping_info': {
|
||||
'type': 'array',
|
||||
'properties': {
|
||||
'domain_name': {
|
||||
'type': 'string',
|
||||
'required': True
|
||||
},
|
||||
'san_cert_name': {
|
||||
'type': 'string',
|
||||
'required': True
|
||||
},
|
||||
},
|
||||
'minItems': 1,
|
||||
},
|
||||
'update_type': {
|
||||
'type': 'string',
|
||||
'enum': ['hostsnames']
|
||||
'enum': ['hostnames']
|
||||
},
|
||||
'action': {
|
||||
'type': 'string',
|
||||
|
|
|
|||
|
|
@ -119,11 +119,50 @@ class SSLCertificateSchema(schema_base.SchemaBase):
|
|||
'minLength': 3,
|
||||
'maxLength': 253
|
||||
},
|
||||
'san_cert_name': {
|
||||
'cert_details': {
|
||||
'type': 'object',
|
||||
'required': True,
|
||||
'additionalProperties': False,
|
||||
'properties': {
|
||||
'Akamai': {
|
||||
'type': 'object',
|
||||
'required': True,
|
||||
'additionalProperties': False,
|
||||
'properties': {
|
||||
'extra_info': {
|
||||
'type': 'object',
|
||||
'required': True,
|
||||
'properties': {
|
||||
'san cert': {
|
||||
'type': 'string',
|
||||
'required': True,
|
||||
'minLength': 3,
|
||||
'maxLength': 253
|
||||
},
|
||||
'akamai_spsId': {
|
||||
'type': 'integer',
|
||||
'required': True
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
'flavor_id': {
|
||||
'type': 'string',
|
||||
'required': True,
|
||||
'minLength': 3,
|
||||
'maxLength': 253
|
||||
'minLength': 1,
|
||||
'maxLength': 256
|
||||
},
|
||||
'project_id': {
|
||||
'type': 'string',
|
||||
'required': True,
|
||||
},
|
||||
'cert_type': {
|
||||
'type': 'string',
|
||||
'required': True,
|
||||
'enum': ['san'],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,10 @@
|
|||
{
|
||||
"akamai_check_and_update_cert_status": {
|
||||
"job_type": "akamai_check_and_update_cert_status",
|
||||
"domain_name": "www.abc.com",
|
||||
"flavor_id": "mock",
|
||||
"cert_type": "san",
|
||||
"project_id": "000"
|
||||
},
|
||||
"akamai_update_papi_property_for_mod_san": {
|
||||
"job_type": "akamai_update_papi_property_for_mod_san",
|
||||
"update_cname_host_mapping_info": [
|
||||
{
|
||||
"domain_name": "www.abc.com",
|
||||
"san_cert_name": "secure1.test_san.com"
|
||||
}
|
||||
],
|
||||
"san_cert_domain_suffix": "test_san.com"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,27 +1,25 @@
|
|||
{
|
||||
"invalid_job_type_name": {
|
||||
"job_type": "nonsense",
|
||||
"flavor_id": "mock"
|
||||
"job_type": "nonsense"
|
||||
},
|
||||
"missing_cert_type": {
|
||||
"check_and_update_cert_with_invalid_param": {
|
||||
"job_type": "akamai_check_and_update_cert_status",
|
||||
"domain_name": "www.abc.com",
|
||||
"flavor_id": "mock",
|
||||
"project_id": "000"
|
||||
"not_allowed": "example_value"
|
||||
},
|
||||
"missing_domain_name": {
|
||||
"job_type": "akamai_check_and_update_cert_status",
|
||||
"cert_type": "san",
|
||||
"flavor_id": "mock",
|
||||
"project_id": "000"
|
||||
},
|
||||
"missing_san_cert_name": {
|
||||
"job_type": "akamai_check_and_update_cert_status",
|
||||
"domain_name": "www.abc.com"
|
||||
},
|
||||
"papi_non_existent_san_cert_cname": {
|
||||
"papi_unsupported_update_type": {
|
||||
"job_type": "akamai_update_papi_property_for_mod_san",
|
||||
"domain_name": "www.abc.com",
|
||||
"san_cert_name": "secure_noexist.test_san.com"
|
||||
"update_type": "unsupported_update_type"
|
||||
},
|
||||
"papi_unsupported_action": {
|
||||
"job_type": "akamai_update_papi_property_for_mod_san",
|
||||
"update_type": "unsupported_action"
|
||||
},
|
||||
"papi_unsupported_property_spec": {
|
||||
"job_type": "akamai_update_papi_property_for_mod_san",
|
||||
"update_type": "unsupported_property_spec"
|
||||
},
|
||||
"papi_unsupported_san_cert_suffix": {
|
||||
"job_type": "akamai_update_papi_property_for_mod_san",
|
||||
"update_type": 1
|
||||
}
|
||||
}
|
||||
|
|
@ -56,12 +56,18 @@ class TestSanMappingList(base.FunctionalTest):
|
|||
def test_put_san_mapping_list_positive(self):
|
||||
put_data = [
|
||||
{
|
||||
"domain_name": "test-san1.cnamecdn.com",
|
||||
"san_cert_name": "san1.sample.com"
|
||||
},
|
||||
{
|
||||
"domain_name": "test-san2.cnamecdn.com",
|
||||
"san_cert_name": "san2.sample.com"
|
||||
"cert_details": {
|
||||
"Akamai": {
|
||||
"extra_info": {
|
||||
"san cert": "secure1.san1.altcdn.com",
|
||||
"akamai_spsId": 1234
|
||||
}
|
||||
}
|
||||
},
|
||||
"project_id": "000",
|
||||
"cert_type": "san",
|
||||
"domain_name": "www.mockssl.com",
|
||||
"flavor_id": "premium"
|
||||
}
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import json
|
|||
import mock
|
||||
from taskflow import engines
|
||||
|
||||
from poppy.model import ssl_certificate
|
||||
from poppy.provider.akamai.background_jobs.check_cert_status_and_update \
|
||||
import check_cert_status_and_update_flow
|
||||
from poppy.provider.akamai.background_jobs.update_property import (
|
||||
|
|
@ -71,10 +72,23 @@ class TestAkamaiBJFlowRuns(base.TestCase):
|
|||
self.addCleanup(bootstrap_patcher.stop)
|
||||
|
||||
def test_check_cert_status_and_update_flow(self):
|
||||
cert_obj_json = json.dumps(
|
||||
ssl_certificate.SSLCertificate(
|
||||
'cdn',
|
||||
'website.com',
|
||||
'san',
|
||||
cert_details={
|
||||
'Akamai': {
|
||||
'extra_info': {
|
||||
'san cert': 'secure1.san1.testcdn.com'
|
||||
}
|
||||
}
|
||||
}
|
||||
).to_dict()
|
||||
)
|
||||
|
||||
kwargs = {
|
||||
'domain_name': "blog.testabc.com",
|
||||
'cert_type': "san",
|
||||
'flavor_id': "premium",
|
||||
'cert_obj_json': cert_obj_json,
|
||||
'project_id': "000"
|
||||
}
|
||||
engines.run(check_cert_status_and_update_flow.
|
||||
|
|
|
|||
Loading…
Reference in New Issue