diff --git a/poppy/manager/default/ssl_certificate.py b/poppy/manager/default/ssl_certificate.py index 7ed8ab96..ded9b9cb 100644 --- a/poppy/manager/default/ssl_certificate.py +++ b/poppy/manager/default/ssl_certificate.py @@ -113,3 +113,40 @@ class DefaultSSLCertificateController(base.SSLCertificateController): "validate_service": r.get('validate_service', True)} for r in res ] + + def update_san_retry_list(self, queue_data_list): + for r in queue_data_list: + service_obj = self.storage_controller\ + .get_service_details_by_domain_name(r['domain_name']) + if service_obj is None and r.get('validate_service', True): + raise LookupError(u'Domain {0} does not exist on any service, ' + 'are you sure you want to proceed request, ' + '{1}? You can set validate_service to False ' + 'to retry this san-retry request forcefully'. + format(r['domain_name'], r)) + + cert_for_domain = self.storage_controller.get_certs_by_domain( + r['domain_name']) + if cert_for_domain != []: + if cert_for_domain.get_cert_status() == "deployed": + raise ValueError(u'Cert on {0} already exists'. + format(r['domain_name'])) + + new_queue_data = [ + json.dumps({'flavor_id': r['flavor_id'], # flavor_id + 'domain_name': r['domain_name'], # domain_name + 'project_id': r['project_id'], + 'validate_service': r.get('validate_service', True)}) + for r in queue_data_list + ] + res, diff = [], [] + if 'akamai' in self._driver.providers: + akamai_driver = self._driver.providers['akamai'].obj + orig = [json.loads(r) for r in + akamai_driver.mod_san_queue.traverse_queue()] + res = [json.loads(r) for r in + akamai_driver.mod_san_queue.put_queue_data(new_queue_data)] + + diff = tuple(x for x in res if x not in orig) + # other provider's retry-list implementaiton goes here + return res, diff diff --git a/poppy/provider/akamai/mod_san_queue/base.py b/poppy/provider/akamai/mod_san_queue/base.py index 0634fa49..334a8d7a 100644 --- a/poppy/provider/akamai/mod_san_queue/base.py +++ b/poppy/provider/akamai/mod_san_queue/base.py @@ -43,5 +43,9 @@ class ModSanQueue(object): '''Travese queue and resturn all items on the queue in a list''' raise NotImplementedError + def put_queue_data(self, queue_data_list): + '''Juggling and put new queue data list in the queue''' + raise NotImplementedError + def move_request_to_top(self): raise NotImplementedError diff --git a/poppy/provider/akamai/mod_san_queue/zookeeper_queue.py b/poppy/provider/akamai/mod_san_queue/zookeeper_queue.py index 6e65e9ef..c42d6636 100644 --- a/poppy/provider/akamai/mod_san_queue/zookeeper_queue.py +++ b/poppy/provider/akamai/mod_san_queue/zookeeper_queue.py @@ -67,6 +67,17 @@ class ZookeeperModSanQueue(base.ModSanQueue): self.mod_san_queue_backend.put_all(res) return res + def put_queue_data(self, queue_data): + # put queue data will replace all existing + # queue data with the incoming new queue_data + # dequeue all the existing data + while len(self.mod_san_queue_backend) > 0: + self.mod_san_queue_backend.get() + self.mod_san_queue_backend.consume() + # put in all the new data + self.mod_san_queue_backend.put_all(queue_data) + return queue_data + def dequeue_mod_san_request(self, consume=True): res = self.mod_san_queue_backend.get() if consume: diff --git a/poppy/storage/mockdb/services.py b/poppy/storage/mockdb/services.py index 24bbe0aa..7c80189d 100644 --- a/poppy/storage/mockdb/services.py +++ b/poppy/storage/mockdb/services.py @@ -171,7 +171,13 @@ class ServicesController(base.ServicesController): self.certs[key].cert_details = cert_details def get_service_details_by_domain_name(self, domain_name): - pass + for service_id in self.created_services: + service_dict_in_cache = self.created_services[service_id] + if domain_name in [d['domain'] + for d in service_dict_in_cache['domains']]: + service_result = self.format_result(service_dict_in_cache) + service_result._status = 'deployed' + return service_result def create_cert(self, project_id, cert_obj): key = (cert_obj.flavor_id, cert_obj.domain_name, cert_obj.cert_type) @@ -181,7 +187,7 @@ class ServicesController(base.ServicesController): raise ValueError def get_certs_by_domain(self, domain_name, project_id=None, flavor_id=None, - cert_type=None): + cert_type=None, status=u'create_in_progress'): certs = [] for cert in self.certs: if domain_name in cert: @@ -203,7 +209,7 @@ class ServicesController(base.ServicesController): ), u'create_at': u'2015-09-29 16:09:12.429147', u'san cert': u'secure2.san1.test_123.com', - u'status': u'create_in_progress'} + u'status': status} } } ) diff --git a/poppy/transport/pecan/controllers/v1/admin.py b/poppy/transport/pecan/controllers/v1/admin.py index 76a065f6..ab51bf60 100644 --- a/poppy/transport/pecan/controllers/v1/admin.py +++ b/poppy/transport/pecan/controllers/v1/admin.py @@ -26,6 +26,7 @@ from poppy.transport.validators.schemas import background_jobs from poppy.transport.validators.schemas import domain_migration from poppy.transport.validators.schemas import service_action from poppy.transport.validators.schemas import service_limit +from poppy.transport.validators.schemas import ssl_certificate from poppy.transport.validators.stoplight import decorators from poppy.transport.validators.stoplight import helpers as stoplight_helpers from poppy.transport.validators.stoplight import rule @@ -122,6 +123,37 @@ class AkamaiRetryListController(base.Controller, hooks.HookController): return res + @pecan.expose('json') + @decorators.validate( + request=rule.Rule( + helpers.json_matches_service_schema( + ssl_certificate.SSLCertificateSchema.get_schema( + "retry_list", "PUT")), + helpers.abort_with_message, + stoplight_helpers.pecan_getter)) + def put(self): + """The input of the queue data must be a list of dictionaries: + + (after json loaded) + [ + { "domain_name": , + "project_id": , + "flavor_id": } + ] + + """ + try: + queue_data = json.loads(pecan.request.body.decode('utf-8')) + res, diff = ( + self._driver.manager.ssl_certificate_controller. + update_san_retry_list(queue_data)) + except Exception as e: + pecan.abort(400, str(e)) + + # result is the new queue, and difference is the difference between the + # new queue and the original one + return {"result": res, "difference": diff} + class AkamaiSSLCertificateController(base.Controller, hooks.HookController): __hooks__ = [poppy_hooks.Context(), poppy_hooks.Error()] diff --git a/poppy/transport/validators/schemas/ssl_certificate.py b/poppy/transport/validators/schemas/ssl_certificate.py index feb98925..4d4f2aa0 100644 --- a/poppy/transport/validators/schemas/ssl_certificate.py +++ b/poppy/transport/validators/schemas/ssl_certificate.py @@ -49,5 +49,37 @@ class SSLCertificateSchema(schema_base.SchemaBase): } } } + }, + + 'retry_list': { + 'PUT': { + 'type': 'array', + "uniqueItems": True, + 'items': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'flavor_id': { + 'type': 'string', + 'required': True, + 'minLength': 1, + 'maxLength': 256 + }, + 'domain_name': { + 'type': 'string', + 'required': True, + 'minLength': 3, + 'maxLength': 253 + }, + 'project_id': { + 'type': 'string', + 'required': True, + }, + 'validate_service': { + 'type': 'boolean' + } + } + } + } } } diff --git a/tests/functional/transport/pecan/base.py b/tests/functional/transport/pecan/base.py index f85c7123..5a1aeab2 100644 --- a/tests/functional/transport/pecan/base.py +++ b/tests/functional/transport/pecan/base.py @@ -44,6 +44,11 @@ class BaseFunctionalTest(base.TestCase): b_obj.distributed_task.job_board = mock.Mock() b_obj.distributed_task.job_board.return_value = ( mock_persistence.copy()) + # Note(tonytan4ever):Need this hack to preserve mockdb storage + # controller's service cache + b_obj.manager.ssl_certificate_controller.storage_controller = ( + b_obj.manager.services_controller.storage_controller + ) poppy_wsgi = b_obj.transport.app self.app = webtest.app.TestApp(poppy_wsgi) diff --git a/tests/functional/transport/pecan/controllers/data_put_retry_list_bad.json b/tests/functional/transport/pecan/controllers/data_put_retry_list_bad.json new file mode 100644 index 00000000..a249ad8a --- /dev/null +++ b/tests/functional/transport/pecan/controllers/data_put_retry_list_bad.json @@ -0,0 +1,36 @@ +{ + "missing_domain_name": [ + { + "project_id": "000", + "flavor_id": "premium" + }, + { + "domain_name": "abc2.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium" + } + ], + + "missing_project_id": [ + { + "domain_name": "abc1.cnamecdn.com", + "flavor_id": "premium" + }, + { + "domain_name": "abc2.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium" + } + ], + + "missing_everything": [ + { + }, + { + "domain_name": "abc2.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium" + } + ] + +} \ No newline at end of file diff --git a/tests/functional/transport/pecan/controllers/test_retry_list.py b/tests/functional/transport/pecan/controllers/test_retry_list.py index d94f4e51..c635a58b 100644 --- a/tests/functional/transport/pecan/controllers/test_retry_list.py +++ b/tests/functional/transport/pecan/controllers/test_retry_list.py @@ -13,10 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools +import json import uuid import ddt +import mock +from poppy.storage.mockdb import services from tests.functional.transport.pecan import base @@ -34,3 +38,216 @@ class TestRetryList(base.FunctionalTest): headers={ 'X-Project-ID': self.project_id}) self.assertEqual(200, response.status_code) + + @ddt.file_data("data_put_retry_list_bad.json") + def test_put_retry_list_negative(self, put_data): + response = self.app.put('/v1.0/admin/provider/akamai/' + 'ssl_certificate/retry_list', + params=json.dumps(put_data), + headers={ + 'Content-Type': 'application/json', + 'X-Project-ID': self.project_id}, + expect_errors=True) + self.assertEqual(400, response.status_code) + + def test_put_retry_list_positive(self): + put_data = [ + { + "domain_name": "test-san1.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium", + "validate_service": False + }, + { + "domain_name": "test-san2.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium", + "validate_service": False + } + ] + response = self.app.put('/v1.0/admin/provider/akamai/' + 'ssl_certificate/retry_list', + params=json.dumps(put_data), + headers={ + 'Content-Type': 'application/json', + 'X-Project-ID': self.project_id}, + ) + self.assertEqual(200, response.status_code) + + def test_put_retry_list_negative_with_validate_service(self): + put_data = [ + { + "domain_name": "test-san1.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium", + "validate_service": False + }, + { + "domain_name": "test-san2.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium", + "validate_service": True + } + ] + response = self.app.put('/v1.0/admin/provider/akamai/' + 'ssl_certificate/retry_list', + params=json.dumps(put_data), + headers={ + 'Content-Type': 'application/json', + 'X-Project-ID': self.project_id}, + expect_errors=True) + self.assertEqual(400, response.status_code) + + def test_put_retry_list_negative_with_deployed_domain(self): + # A cert already in deployed status will cause 400. + with mock.patch('poppy.storage.mockdb.services.ServicesController.' + 'get_certs_by_domain', + new=functools. + partial(services.ServicesController. + get_certs_by_domain, + status='deployed')): + self.service_name = str(uuid.uuid1()) + self.flavor_id = str(uuid.uuid1()) + + # create a mock flavor to be used by new service creations + flavor_json = { + "id": self.flavor_id, + "providers": [ + { + "provider": "mock", + "links": [ + { + "href": "http://mock.cdn", + "rel": "provider_url" + } + ] + } + ] + } + response = self.app.post('/v1.0/flavors', + params=json.dumps(flavor_json), + headers={ + "Content-Type": "application/json", + "X-Project-ID": self.project_id}) + + self.assertEqual(201, response.status_code) + + # create a service with the domain_name test-san2.cnamecdn.com + self.service_json = { + "name": self.service_name, + "domains": [ + {"domain": "test-san2.cnamecdn.com", + "protocol": "https", + "certificate": "san"} + ], + "origins": [ + { + "origin": "mocksite.com", + } + ], + "flavor_id": self.flavor_id, + } + + response = self.app.post('/v1.0/services', + params=json.dumps(self.service_json), + headers={ + 'Content-Type': 'application/json', + 'X-Project-ID': self.project_id}) + self.assertEqual(202, response.status_code) + + put_data = [ + { + "domain_name": "test-san1.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium", + "validate_service": False + }, + { + "domain_name": "test-san2.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium", + "validate_service": True + } + ] + response = self.app.put('/v1.0/admin/provider/akamai/' + 'ssl_certificate/retry_list', + params=json.dumps(put_data), + headers={ + 'Content-Type': 'application/json', + 'X-Project-ID': self.project_id}, + expect_errors=True) + self.assertEqual(400, response.status_code) + + def test_put_retry_list_positive_with_validate_service(self): + self.service_name = str(uuid.uuid1()) + self.flavor_id = str(uuid.uuid1()) + + # create a mock flavor to be used by new service creations + flavor_json = { + "id": self.flavor_id, + "providers": [ + { + "provider": "mock", + "links": [ + { + "href": "http://mock.cdn", + "rel": "provider_url" + } + ] + } + ] + } + response = self.app.post('/v1.0/flavors', + params=json.dumps(flavor_json), + headers={ + "Content-Type": "application/json", + "X-Project-ID": self.project_id}) + + self.assertEqual(201, response.status_code) + + # create a service with the domain_name test-san2.cnamecdn.com + self.service_json = { + "name": self.service_name, + "domains": [ + {"domain": "test-san2.cnamecdn.com", + "protocol": "https", + "certificate": "san"} + ], + "origins": [ + { + "origin": "mocksite.com", + } + ], + "flavor_id": self.flavor_id, + } + + response = self.app.post('/v1.0/services', + params=json.dumps(self.service_json), + headers={ + 'Content-Type': 'application/json', + 'X-Project-ID': self.project_id}) + self.assertEqual(202, response.status_code) + + # This time the service is present, so the request goes thru + put_data = [ + { + "domain_name": "test-san1.cnamecdn.com", + "project_id": "000", + "flavor_id": "premium", + "validate_service": False + }, + { + "domain_name": "test-san2.cnamecdn.com", + "project_id": self.project_id, + "flavor_id": "premium", + "validate_service": True + } + ] + response = self.app.put('/v1.0/admin/provider/akamai/' + 'ssl_certificate/retry_list', + params=json.dumps(put_data), + headers={ + 'Content-Type': 'application/json', + 'X-Project-ID': self.project_id}, + expect_errors=True) + self.assertEqual(200, response.status_code) diff --git a/tests/unit/provider/akamai/test_mod_san_queue.py b/tests/unit/provider/akamai/test_mod_san_queue.py index cd30033f..2373c1a5 100644 --- a/tests/unit/provider/akamai/test_mod_san_queue.py +++ b/tests/unit/provider/akamai/test_mod_san_queue.py @@ -112,3 +112,22 @@ class TestModSanQueue(base.TestCase): self.assertTrue(len(res) == 10) res = [json.loads(r.decode('utf-8')) for r in res] self.assertTrue(res == cert_obj_list) + + def test_put_queue_data(self): + res = self.zk_queue.put_queue_data([]) + self.assertTrue(len(res) == 0) + + cert_obj_list = [] + for i in range(10): + cert_obj = { + "cert_type": "san", + "domain_name": "www.abc%s.com" % str(i), + "flavor_id": "premium" + } + cert_obj_list.append(cert_obj) + self.zk_queue.put_queue_data( + [json.dumps(o).encode('utf-8') for o in cert_obj_list]) + self.assertTrue(len(self.zk_queue.mod_san_queue_backend) == 10) + res = self.zk_queue.traverse_queue() + res = [json.loads(r.decode('utf-8')) for r in res] + self.assertTrue(res == cert_obj_list)