feat: PUT Retry-list endpoint

Partially Implements: blueprint san-retry-list

Change-Id: I746c27455081459102121b11fe57a2bf5960c7b0

poppy/manager/default/ssl_certificate.py

@@ -113,3 +113,40 @@ class DefaultSSLCertificateController(base.SSLCertificateController):
              "validate_service": r.get('validate_service', True)}
             for r in res
         ]
+
+    def update_san_retry_list(self, queue_data_list):
+        for r in queue_data_list:
+            service_obj = self.storage_controller\
+                .get_service_details_by_domain_name(r['domain_name'])
+            if service_obj is None and r.get('validate_service', True):
+                raise LookupError(u'Domain {0} does not exist on any service, '
+                                  'are you sure you want to proceed request, '
+                                  '{1}? You can set validate_service to False '
+                                  'to retry this san-retry request forcefully'.
+                                  format(r['domain_name'], r))
+
+            cert_for_domain = self.storage_controller.get_certs_by_domain(
+                r['domain_name'])
+            if cert_for_domain != []:
+                if cert_for_domain.get_cert_status() == "deployed":
+                    raise ValueError(u'Cert on {0} already exists'.
+                                     format(r['domain_name']))
+
+        new_queue_data = [
+            json.dumps({'flavor_id':   r['flavor_id'],  # flavor_id
+                        'domain_name': r['domain_name'],    # domain_name
+                        'project_id': r['project_id'],
+                        'validate_service': r.get('validate_service', True)})
+            for r in queue_data_list
+        ]
+        res, diff = [], []
+        if 'akamai' in self._driver.providers:
+            akamai_driver = self._driver.providers['akamai'].obj
+            orig = [json.loads(r) for r in
+                    akamai_driver.mod_san_queue.traverse_queue()]
+            res = [json.loads(r) for r in
+                   akamai_driver.mod_san_queue.put_queue_data(new_queue_data)]
+
+            diff = tuple(x for x in res if x not in orig)
+        # other provider's retry-list implementaiton goes here
+        return res, diff

poppy/provider/akamai/mod_san_queue/base.py

@@ -43,5 +43,9 @@ class ModSanQueue(object):
         '''Travese queue and resturn all items on the queue in a list'''
         raise NotImplementedError
 
+    def put_queue_data(self, queue_data_list):
+        '''Juggling and put new queue data list in the queue'''
+        raise NotImplementedError
+
     def move_request_to_top(self):
         raise NotImplementedError

poppy/provider/akamai/mod_san_queue/zookeeper_queue.py

@@ -67,6 +67,17 @@ class ZookeeperModSanQueue(base.ModSanQueue):
         self.mod_san_queue_backend.put_all(res)
         return res
 
+    def put_queue_data(self, queue_data):
+        # put queue data will replace all existing
+        # queue data with the incoming new queue_data
+        # dequeue all the existing data
+        while len(self.mod_san_queue_backend) > 0:
+            self.mod_san_queue_backend.get()
+            self.mod_san_queue_backend.consume()
+        # put in all the new data
+        self.mod_san_queue_backend.put_all(queue_data)
+        return queue_data
+
     def dequeue_mod_san_request(self, consume=True):
         res = self.mod_san_queue_backend.get()
         if consume:

poppy/storage/mockdb/services.py

@@ -171,7 +171,13 @@ class ServicesController(base.ServicesController):
             self.certs[key].cert_details = cert_details
 
     def get_service_details_by_domain_name(self, domain_name):
-        pass
+        for service_id in self.created_services:
+            service_dict_in_cache = self.created_services[service_id]
+            if domain_name in [d['domain']
+                               for d in service_dict_in_cache['domains']]:
+                service_result = self.format_result(service_dict_in_cache)
+                service_result._status = 'deployed'
+                return service_result
 
     def create_cert(self, project_id, cert_obj):
         key = (cert_obj.flavor_id, cert_obj.domain_name, cert_obj.cert_type)
@@ -181,7 +187,7 @@ class ServicesController(base.ServicesController):
             raise ValueError
 
     def get_certs_by_domain(self, domain_name, project_id=None, flavor_id=None,
-                            cert_type=None):
+                            cert_type=None, status=u'create_in_progress'):
         certs = []
         for cert in self.certs:
             if domain_name in cert:
@@ -203,7 +209,7 @@ class ServicesController(base.ServicesController):
                                                      ),
                                 u'create_at': u'2015-09-29 16:09:12.429147',
                                 u'san cert': u'secure2.san1.test_123.com',
-                                u'status': u'create_in_progress'}
+                                u'status': status}
                             }
                     }
                 )

poppy/transport/pecan/controllers/v1/admin.py

@@ -26,6 +26,7 @@ from poppy.transport.validators.schemas import background_jobs
 from poppy.transport.validators.schemas import domain_migration
 from poppy.transport.validators.schemas import service_action
 from poppy.transport.validators.schemas import service_limit
+from poppy.transport.validators.schemas import ssl_certificate
 from poppy.transport.validators.stoplight import decorators
 from poppy.transport.validators.stoplight import helpers as stoplight_helpers
 from poppy.transport.validators.stoplight import rule
@@ -122,6 +123,37 @@ class AkamaiRetryListController(base.Controller, hooks.HookController):
 
         return res
 
+    @pecan.expose('json')
+    @decorators.validate(
+        request=rule.Rule(
+            helpers.json_matches_service_schema(
+                ssl_certificate.SSLCertificateSchema.get_schema(
+                    "retry_list", "PUT")),
+            helpers.abort_with_message,
+            stoplight_helpers.pecan_getter))
+    def put(self):
+        """The input of the queue data must be a list of dictionaries:
+
+        (after json loaded)
+        [
+          { "domain_name": <domain_name>,
+            "project_id": <project_id>,
+            "flavor_id": <flavor_id> }
+        ]
+
+        """
+        try:
+            queue_data = json.loads(pecan.request.body.decode('utf-8'))
+            res, diff = (
+                self._driver.manager.ssl_certificate_controller.
+                update_san_retry_list(queue_data))
+        except Exception as e:
+            pecan.abort(400, str(e))
+
+        # result is the new queue, and difference is the difference between the
+        # new queue and the original one
+        return {"result": res,  "difference": diff}
+
 
 class AkamaiSSLCertificateController(base.Controller, hooks.HookController):
     __hooks__ = [poppy_hooks.Context(), poppy_hooks.Error()]

poppy/transport/validators/schemas/ssl_certificate.py

@@ -49,5 +49,37 @@ class SSLCertificateSchema(schema_base.SchemaBase):
                     }
                 }
             }
+        },
+
+        'retry_list': {
+            'PUT': {
+                'type': 'array',
+                "uniqueItems": True,
+                'items': {
+                    'type': 'object',
+                    'additionalProperties': False,
+                    'properties': {
+                        'flavor_id': {
+                            'type': 'string',
+                            'required': True,
+                            'minLength': 1,
+                            'maxLength': 256
+                        },
+                        'domain_name': {
+                            'type': 'string',
+                            'required': True,
+                            'minLength': 3,
+                            'maxLength': 253
+                        },
+                        'project_id': {
+                            'type': 'string',
+                            'required': True,
+                        },
+                        'validate_service': {
+                            'type': 'boolean'
+                        }
+                    }
+                }
+            }
         }
     }

tests/functional/transport/pecan/base.py

@@ -44,6 +44,11 @@ class BaseFunctionalTest(base.TestCase):
         b_obj.distributed_task.job_board = mock.Mock()
         b_obj.distributed_task.job_board.return_value = (
             mock_persistence.copy())
+        # Note(tonytan4ever):Need this hack to preserve mockdb storage
+        # controller's service cache
+        b_obj.manager.ssl_certificate_controller.storage_controller = (
+            b_obj.manager.services_controller.storage_controller
+        )
         poppy_wsgi = b_obj.transport.app
 
         self.app = webtest.app.TestApp(poppy_wsgi)

tests/functional/transport/pecan/controllers/data_put_retry_list_bad.json

@@ -0,0 +1,36 @@
+{
+    "missing_domain_name": [
+       {
+         "project_id": "000",
+         "flavor_id": "premium"
+       },
+       {
+         "domain_name": "abc2.cnamecdn.com",
+         "project_id": "000",
+         "flavor_id": "premium"
+        }
+    ],
+
+    "missing_project_id": [
+      {
+         "domain_name": "abc1.cnamecdn.com",
+         "flavor_id": "premium"
+      },
+      {
+         "domain_name": "abc2.cnamecdn.com",
+         "project_id": "000",
+         "flavor_id": "premium"
+      }
+    ],
+
+    "missing_everything": [
+      {
+      },
+      {
+        "domain_name": "abc2.cnamecdn.com",
+        "project_id": "000",
+        "flavor_id": "premium"
+      }
+    ]
+
+}

tests/functional/transport/pecan/controllers/test_retry_list.py

@@ -13,10 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
+import json
 import uuid
 
 import ddt
+import mock
 
+from poppy.storage.mockdb import services
 from tests.functional.transport.pecan import base
 
 
@@ -34,3 +38,216 @@ class TestRetryList(base.FunctionalTest):
                                 headers={
                                     'X-Project-ID': self.project_id})
         self.assertEqual(200, response.status_code)
+
+    @ddt.file_data("data_put_retry_list_bad.json")
+    def test_put_retry_list_negative(self, put_data):
+        response = self.app.put('/v1.0/admin/provider/akamai/'
+                                'ssl_certificate/retry_list',
+                                params=json.dumps(put_data),
+                                headers={
+                                    'Content-Type': 'application/json',
+                                    'X-Project-ID': self.project_id},
+                                expect_errors=True)
+        self.assertEqual(400, response.status_code)
+
+    def test_put_retry_list_positive(self):
+        put_data = [
+            {
+                "domain_name": "test-san1.cnamecdn.com",
+                "project_id": "000",
+                "flavor_id": "premium",
+                "validate_service": False
+            },
+            {
+                "domain_name": "test-san2.cnamecdn.com",
+                "project_id": "000",
+                "flavor_id": "premium",
+                "validate_service": False
+            }
+        ]
+        response = self.app.put('/v1.0/admin/provider/akamai/'
+                                'ssl_certificate/retry_list',
+                                params=json.dumps(put_data),
+                                headers={
+                                    'Content-Type': 'application/json',
+                                    'X-Project-ID': self.project_id},
+                                )
+        self.assertEqual(200, response.status_code)
+
+    def test_put_retry_list_negative_with_validate_service(self):
+        put_data = [
+            {
+                "domain_name": "test-san1.cnamecdn.com",
+                "project_id": "000",
+                "flavor_id": "premium",
+                "validate_service": False
+            },
+            {
+                "domain_name": "test-san2.cnamecdn.com",
+                "project_id": "000",
+                "flavor_id": "premium",
+                "validate_service": True
+            }
+        ]
+        response = self.app.put('/v1.0/admin/provider/akamai/'
+                                'ssl_certificate/retry_list',
+                                params=json.dumps(put_data),
+                                headers={
+                                    'Content-Type': 'application/json',
+                                    'X-Project-ID': self.project_id},
+                                expect_errors=True)
+        self.assertEqual(400, response.status_code)
+
+    def test_put_retry_list_negative_with_deployed_domain(self):
+        # A cert already in deployed status will cause 400.
+        with mock.patch('poppy.storage.mockdb.services.ServicesController.'
+                        'get_certs_by_domain',
+                        new=functools.
+                        partial(services.ServicesController.
+                                get_certs_by_domain,
+                                status='deployed')):
+            self.service_name = str(uuid.uuid1())
+            self.flavor_id = str(uuid.uuid1())
+
+            # create a mock flavor to be used by new service creations
+            flavor_json = {
+                "id": self.flavor_id,
+                "providers": [
+                    {
+                        "provider": "mock",
+                        "links": [
+                            {
+                                "href": "http://mock.cdn",
+                                "rel": "provider_url"
+                            }
+                        ]
+                    }
+                ]
+            }
+            response = self.app.post('/v1.0/flavors',
+                                     params=json.dumps(flavor_json),
+                                     headers={
+                                         "Content-Type": "application/json",
+                                         "X-Project-ID": self.project_id})
+
+            self.assertEqual(201, response.status_code)
+
+            # create a service with the domain_name test-san2.cnamecdn.com
+            self.service_json = {
+                "name": self.service_name,
+                "domains": [
+                    {"domain": "test-san2.cnamecdn.com",
+                     "protocol": "https",
+                     "certificate": "san"}
+                ],
+                "origins": [
+                    {
+                        "origin": "mocksite.com",
+                    }
+                ],
+                "flavor_id": self.flavor_id,
+            }
+
+            response = self.app.post('/v1.0/services',
+                                     params=json.dumps(self.service_json),
+                                     headers={
+                                         'Content-Type': 'application/json',
+                                         'X-Project-ID': self.project_id})
+            self.assertEqual(202, response.status_code)
+
+            put_data = [
+                {
+                    "domain_name": "test-san1.cnamecdn.com",
+                    "project_id": "000",
+                    "flavor_id": "premium",
+                    "validate_service": False
+                },
+                {
+                    "domain_name": "test-san2.cnamecdn.com",
+                    "project_id": "000",
+                    "flavor_id": "premium",
+                    "validate_service": True
+                }
+            ]
+            response = self.app.put('/v1.0/admin/provider/akamai/'
+                                    'ssl_certificate/retry_list',
+                                    params=json.dumps(put_data),
+                                    headers={
+                                        'Content-Type': 'application/json',
+                                        'X-Project-ID': self.project_id},
+                                    expect_errors=True)
+            self.assertEqual(400, response.status_code)
+
+    def test_put_retry_list_positive_with_validate_service(self):
+        self.service_name = str(uuid.uuid1())
+        self.flavor_id = str(uuid.uuid1())
+
+        # create a mock flavor to be used by new service creations
+        flavor_json = {
+            "id": self.flavor_id,
+            "providers": [
+                {
+                    "provider": "mock",
+                    "links": [
+                        {
+                            "href": "http://mock.cdn",
+                            "rel": "provider_url"
+                        }
+                    ]
+                }
+            ]
+        }
+        response = self.app.post('/v1.0/flavors',
+                                 params=json.dumps(flavor_json),
+                                 headers={
+                                     "Content-Type": "application/json",
+                                     "X-Project-ID": self.project_id})
+
+        self.assertEqual(201, response.status_code)
+
+        # create a service with the domain_name test-san2.cnamecdn.com
+        self.service_json = {
+            "name": self.service_name,
+            "domains": [
+                {"domain": "test-san2.cnamecdn.com",
+                 "protocol": "https",
+                 "certificate": "san"}
+            ],
+            "origins": [
+                {
+                    "origin": "mocksite.com",
+                }
+            ],
+            "flavor_id": self.flavor_id,
+        }
+
+        response = self.app.post('/v1.0/services',
+                                 params=json.dumps(self.service_json),
+                                 headers={
+                                     'Content-Type': 'application/json',
+                                     'X-Project-ID': self.project_id})
+        self.assertEqual(202, response.status_code)
+
+        # This time the service is present, so the request goes thru
+        put_data = [
+            {
+                "domain_name": "test-san1.cnamecdn.com",
+                "project_id": "000",
+                "flavor_id": "premium",
+                "validate_service": False
+            },
+            {
+                "domain_name": "test-san2.cnamecdn.com",
+                "project_id": self.project_id,
+                "flavor_id": "premium",
+                "validate_service": True
+            }
+        ]
+        response = self.app.put('/v1.0/admin/provider/akamai/'
+                                'ssl_certificate/retry_list',
+                                params=json.dumps(put_data),
+                                headers={
+                                    'Content-Type': 'application/json',
+                                    'X-Project-ID': self.project_id},
+                                expect_errors=True)
+        self.assertEqual(200, response.status_code)

tests/unit/provider/akamai/test_mod_san_queue.py

@@ -112,3 +112,22 @@ class TestModSanQueue(base.TestCase):
         self.assertTrue(len(res) == 10)
         res = [json.loads(r.decode('utf-8')) for r in res]
         self.assertTrue(res == cert_obj_list)
+
+    def test_put_queue_data(self):
+        res = self.zk_queue.put_queue_data([])
+        self.assertTrue(len(res) == 0)
+
+        cert_obj_list = []
+        for i in range(10):
+            cert_obj = {
+                "cert_type": "san",
+                "domain_name": "www.abc%s.com" % str(i),
+                "flavor_id": "premium"
+            }
+            cert_obj_list.append(cert_obj)
+        self.zk_queue.put_queue_data(
+            [json.dumps(o).encode('utf-8') for o in cert_obj_list])
+        self.assertTrue(len(self.zk_queue.mod_san_queue_backend) == 10)
+        res = self.zk_queue.traverse_queue()
+        res = [json.loads(r.decode('utf-8')) for r in res]
+        self.assertTrue(res == cert_obj_list)