diff --git a/poppy/distributed_task/taskflow/task/create_service_tasks.py b/poppy/distributed_task/taskflow/task/create_service_tasks.py index 23cd9d30..8893977b 100644 --- a/poppy/distributed_task/taskflow/task/create_service_tasks.py +++ b/poppy/distributed_task/taskflow/task/create_service_tasks.py @@ -52,6 +52,13 @@ class CreateProviderServicesTask(task.Task): providers_list = json.loads(providers_list_json) try: service_obj = self.storage_controller.get(project_id, service_id) + for domain in service_obj.domains: + if domain.certificate == 'san': + cert_for_domain = ( + self.storage_controller.get_cert_by_domain( + domain.domain, domain.certificate, + service_obj.flavor_id, project_id)) + domain.cert_info = cert_for_domain except ValueError: msg = 'Creating service {0} from Poppy failed. ' \ 'No such service exists'.format(service_id) @@ -203,6 +210,8 @@ class GatherProviderDetailsTask(task.Task): for responder in responders: for provider_name in responder: error_class = None + domains_certificate_status = responder[provider_name].get( + 'domains_certificate_status', {}) if 'error' in responder[provider_name]: error_msg = responder[provider_name]['error'] error_info = responder[provider_name]['error_detail'] @@ -213,6 +222,8 @@ class GatherProviderDetailsTask(task.Task): provider_details.ProviderDetail( error_info=error_info, status='failed', + domains_certificate_status=( + domains_certificate_status), error_message=error_msg, error_class=error_class)) elif 'error' in dns_responder[provider_name]: @@ -225,6 +236,8 @@ class GatherProviderDetailsTask(task.Task): provider_details.ProviderDetail( error_info=error_info, status='failed', + domains_certificate_status=( + domains_certificate_status), error_message=error_msg, error_class=error_class)) else: @@ -234,6 +247,8 @@ class GatherProviderDetailsTask(task.Task): provider_details_dict[provider_name] = ( provider_details.ProviderDetail( provider_service_id=responder[provider_name]['id'], + domains_certificate_status=( + domains_certificate_status), access_urls=access_urls)) if 'status' in responder[provider_name]: diff --git a/poppy/distributed_task/taskflow/task/update_service_tasks.py b/poppy/distributed_task/taskflow/task/update_service_tasks.py index c9250442..76fd8b16 100644 --- a/poppy/distributed_task/taskflow/task/update_service_tasks.py +++ b/poppy/distributed_task/taskflow/task/update_service_tasks.py @@ -195,11 +195,15 @@ class GatherProviderDetailsTask(task.Task): provider_details_dict = {} for responder in responders: for provider_name in responder: + domains_certificate_status = responder[provider_name].get( + 'domains_certificate_status', {}) if 'error' in responder[provider_name]: error_flag = True provider_details_dict[provider_name] = ( provider_details.ProviderDetail( status='failed', + domains_certificate_status=( + domains_certificate_status), error_message=responder[provider_name]['error'], error_info=responder[provider_name]['error_detail'] )) @@ -215,6 +219,8 @@ class GatherProviderDetailsTask(task.Task): provider_details.ProviderDetail( error_info=error_info, status='failed', + domains_certificate_status=( + domains_certificate_status), error_message=error_msg, error_class=error_class)) else: @@ -227,10 +233,8 @@ class GatherProviderDetailsTask(task.Task): provider_details_dict[provider_name] = ( provider_details.ProviderDetail( provider_service_id=responder[provider_name]['id'], - access_urls=access_urls)) - provider_details_dict[provider_name] = ( - provider_details.ProviderDetail( - provider_service_id=responder[provider_name]['id'], + domains_certificate_status=( + domains_certificate_status), access_urls=access_urls)) if 'status' in responder[provider_name]: provider_details_dict[provider_name].status = ( @@ -296,6 +300,12 @@ class UpdateProviderDetailsTask_Errors(task.Task): # update the provider details service_obj.provider_details = provider_details_dict + for domain in service_obj.domains: + if hasattr(domain, 'cert_info'): + # we don't want store cert_info in database + # just generate it on demand + delattr(domain, 'cert_info') + # update the service object LOG.info("Service to be updated to {0} " "for project_id: {1} " diff --git a/poppy/manager/default/services.py b/poppy/manager/default/services.py index e6bd009e..fd77986f 100644 --- a/poppy/manager/default/services.py +++ b/poppy/manager/default/services.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import json import random import uuid @@ -29,6 +30,7 @@ from poppy.manager import base from poppy.model.helpers import cachingrule from poppy.model.helpers import rule from poppy.model import service +from poppy.model import ssl_certificate from poppy.openstack.common import log from poppy.transport.validators import helpers as validators from poppy.transport.validators.schemas import service as service_schema @@ -185,6 +187,7 @@ class DefaultServicesController(base.ServicesController): raise e except ValueError as e: raise e + try: self.storage_controller.create(project_id, service_obj) @@ -244,6 +247,10 @@ class DefaultServicesController(base.ServicesController): del service_old_json['operator_status'] del service_old_json['provider_details'] + for domain in service_old_json['domains']: + if 'cert_info' in domain: + del domain['cert_info'] + service_new_json = jsonpatch.apply_patch( service_old_json, service_updates) @@ -265,21 +272,85 @@ class DefaultServicesController(base.ServicesController): service_new = service.Service.init_from_dict(project_id, service_new_json) store = str(uuid.uuid4()).replace('-', '_') - # fixing the old and new shared ssl domains in service_new + service_new.provider_details = service_old.provider_details + # fixing the old and new shared ssl domains in service_new for domain in service_new.domains: - if domain.protocol == 'https' and domain.certificate == 'shared': - customer_domain = domain.domain.split('.')[0] - # if this domain is from service_old - if customer_domain in existing_shared_domains: - domain.domain = existing_shared_domains[customer_domain] - else: - domain.domain = self._pick_shared_ssl_domain( - customer_domain, - service_new.service_id, - store) + if domain.protocol == 'https': + if domain.certificate == 'shared': + customer_domain = domain.domain.split('.')[0] + # if this domain is from service_old + if customer_domain in existing_shared_domains: + domain.domain = existing_shared_domains[ + customer_domain + ] + else: + domain.domain = self._pick_shared_ssl_domain( + customer_domain, + service_new.service_id, + store) + + elif domain.certificate == 'san': + cert_for_domain = ( + self.storage_controller.get_cert_by_domain( + domain.domain, domain.certificate, + service_new.flavor_id, project_id)) + domain.cert_info = cert_for_domain + + # retrofit the access url info into + # certificate_info table + # Note(tonytan4ever): this is for backward + # compatibility + if domain.cert_info is None and \ + service_new.provider_details is not None: + # Note(tonytan4ever): right now we assume + # only one provider per flavor, that's + # why we use values()[0] + access_url_for_domain = ( + service_new.provider_details.values()[0]. + get_domain_access_url(domain.domain)) + if access_url_for_domain is not None: + providers = ( + self.flavor_controller.get( + service_new.flavor_id).providers + ) + san_cert_url = access_url_for_domain.get( + 'provider_url') + # Note(tonytan4ever): stored san_cert_url + # for two times, that's intentional + # a little extra info does not hurt + new_cert_detail = { + providers[0].provider_id.title(): + json.dumps(dict( + cert_domain=san_cert_url, + extra_info={ + 'status': 'deployed', + 'san cert': san_cert_url, + 'created_at': str( + datetime.datetime.now()) + } + )) + } + new_cert_obj = ssl_certificate.SSLCertificate( + service_new.flavor_id, + domain.domain, + 'san', + new_cert_detail + ) + self.storage_controller.create_cert( + project_id, + new_cert_obj + ) + # deserialize cert_details dict + new_cert_obj.cert_details[ + providers[0].provider_id.title()] = json.loads( + new_cert_obj.cert_details[ + providers[0].provider_id.title()] + ) + domain.cert_info = new_cert_obj if hasattr(self, store): delattr(self, store) + # check if the service domain names already exist # existing ones does not count! for d in service_new.domains: diff --git a/poppy/model/helpers/domain.py b/poppy/model/helpers/domain.py index 6766547a..09073c70 100644 --- a/poppy/model/helpers/domain.py +++ b/poppy/model/helpers/domain.py @@ -120,3 +120,11 @@ class Domain(common.DictSerializableModel): if o.protocol == 'https': o.certificate = dict_obj.get("certificate", None) return o + + def to_dict(self): + res = super(Domain, self).to_dict() + # cert info is a temporary property when + # trying to create cert, so skip serialization + if 'cert_info' in res: + res['cert_info'] = res['cert_info'].to_dict() + return res diff --git a/poppy/model/helpers/provider_details.py b/poppy/model/helpers/provider_details.py index 5f58a77e..7b020bae 100644 --- a/poppy/model/helpers/provider_details.py +++ b/poppy/model/helpers/provider_details.py @@ -132,7 +132,7 @@ class ProviderDetail(common.DictSerializableModel): @domains_certificate_status.setter def domains_certificate_status(self, value): - self._domains_certificate_status = value + self._domains_certificate_status = DomainCertificatesStatus(value) @property def error_message(self): @@ -150,12 +150,24 @@ class ProviderDetail(common.DictSerializableModel): def error_class(self, value): self._error_class = value + def get_domain_access_url(self, domain): + '''Find an access url of a domain. + + :param domain + ''' + for access_url in self.access_urls: + if access_url['domain'] == domain: + return access_url + return None + def to_dict(self): result = collections.OrderedDict() result["id"] = self.provider_service_id result["access_urls"] = self.access_urls result["status"] = self.status result["name"] = self.name + result["domains_certificate_status"] = ( + self.domains_certificate_status.to_dict()) result["error_info"] = self.error_info result["error_message"] = self.error_message result["error_class"] = self.error_class @@ -176,6 +188,8 @@ class ProviderDetail(common.DictSerializableModel): "unknown_id") o.access_urls = dict_obj.get("access_urls", []) o.status = dict_obj.get("status", u"deploy_in_progress") + o.domains_certificate_status = dict_obj.get( + "domains_certificate_status", {}) o.name = dict_obj.get("name", None) o.error_info = dict_obj.get("error_info", None) o.error_message = dict_obj.get("error_message", None) diff --git a/poppy/model/ssl_certificate.py b/poppy/model/ssl_certificate.py index 019302b4..34465ec1 100644 --- a/poppy/model/ssl_certificate.py +++ b/poppy/model/ssl_certificate.py @@ -17,6 +17,11 @@ from poppy.model import common VALID_CERT_TYPES = [u'san', u'custom'] +VALID_STATUS_IN_CERT_DETAIL = [ + u'deployed', + u'create_in_progress', + u'failed' +] class SSLCertificate(common.DictSerializableModel): @@ -26,10 +31,12 @@ class SSLCertificate(common.DictSerializableModel): def __init__(self, flavor_id, domain_name, - cert_type): + cert_type, + cert_details={}): self._flavor_id = flavor_id self._domain_name = domain_name self._cert_type = cert_type + self._cert_details = cert_details @property def flavor_id(self): @@ -42,7 +49,7 @@ class SSLCertificate(common.DictSerializableModel): @property def domain_name(self): - """Get service id.""" + """Get domain name""" return self._domain_name @domain_name.setter @@ -51,7 +58,7 @@ class SSLCertificate(common.DictSerializableModel): @property def cert_type(self): - """Get service id.""" + """Get cert type.""" return self._cert_type @cert_type.setter @@ -64,3 +71,48 @@ class SSLCertificate(common.DictSerializableModel): value, VALID_CERT_TYPES) ) + + @property + def cert_details(self): + """Get cert_details.""" + return self._cert_details + + @cert_details.setter + def cert_details(self, value): + """Set cert details.""" + self._cert_type = value + + def get_cert_status(self): + if self.cert_details is None or self.cert_details == {}: + return "deployed" + # Note(tonytan4ever): Right now we assume there is only one + # provider per flavor (that is akamai), so the first one + # value of this dictionary is akamai cert_details + first_provider_cert_details = ( + self.cert_details.values()[0].get("extra_info", None)) + if first_provider_cert_details is None: + return "deployed" + else: + result = first_provider_cert_details.get('status', "deployed") + if result not in VALID_STATUS_IN_CERT_DETAIL: + raise ValueError( + u'Status in cert_details: {0} not in valid options: {1}'. + format( + result, + VALID_STATUS_IN_CERT_DETAIL + ) + ) + return result + + def get_san_edge_name(self): + if self.cert_type == 'san': + if self.cert_details is None or self.cert_details == {}: + return None + first_provider_cert_details = ( + self.cert_details.values()[0].get("extra_info", None)) + if first_provider_cert_details is None: + return None + else: + return first_provider_cert_details.get('san cert', None) + else: + return None diff --git a/poppy/provider/akamai/services.py b/poppy/provider/akamai/services.py index aab7413a..ef4c2510 100644 --- a/poppy/provider/akamai/services.py +++ b/poppy/provider/akamai/services.py @@ -94,6 +94,7 @@ class ServiceController(base.ServiceBase): # a list to represent provide_detail id ids = [] links = [] + domains_certificate_status = {} for classified_domain in classified_domains: # assign the content realm to be the digital property field # of each group @@ -128,7 +129,16 @@ class ServiceController(base.ServiceBase): # pick a san cert for this domain edge_host_name = None if classified_domain.certificate == 'san': - edge_host_name = self._pick_san_edgename() + cert_info = getattr(classified_domain, 'cert_info', None) + if cert_info is None: + continue + else: + edge_host_name = ( + classified_domain.cert_info.get_san_edge_name()) + if edge_host_name is None: + continue + domains_certificate_status[classified_domain.domain] \ + = (classified_domain.cert_info.get_cert_status()) provider_access_url = self._get_provider_access_url( classified_domain, dp, edge_host_name) links.append({'href': provider_access_url, @@ -143,7 +153,9 @@ class ServiceController(base.ServiceBase): return self.responder.failed( "failed to create service - %s" % str(e)) else: - return self.responder.created(json.dumps(ids), links) + return self.responder.created( + json.dumps(ids), links, + domains_certificate_status=domains_certificate_status) def get(self, service_name): pass @@ -171,6 +183,7 @@ class ServiceController(base.ServiceBase): ids = [] links = [] + domains_certificate_status = {} if len(service_obj.domains) > 0: # in this case we need to copy # and tweak the content of one old policy @@ -286,7 +299,19 @@ class ServiceController(base.ServiceBase): 'complete' % (dp, classified_domain.domain)) edge_host_name = None if classified_domain.certificate == 'san': - edge_host_name = self._pick_san_edgename() + cert_info = getattr(classified_domain, 'cert_info', + None) + if cert_info is None: + continue + else: + edge_host_name = ( + classified_domain.cert_info. + get_san_edge_name()) + if edge_host_name is None: + continue + domains_certificate_status[classified_domain.domain] \ + = ( + classified_domain.cert_info.get_cert_status()) provider_access_url = self._get_provider_access_url( classified_domain, dp, edge_host_name) links.append({'href': provider_access_url, @@ -378,7 +403,18 @@ class ServiceController(base.ServiceBase): # This part may need to revisit edge_host_name = None if policy['certificate'] == 'san': - edge_host_name = self._pick_san_edgename() + cert_info = policy.get('cert_info', None) + if cert_info is None: + continue + else: + edge_host_name = ( + classified_domain.cert_info. + get_san_edge_name()) + if edge_host_name is None: + continue + domains_certificate_status[policy['policy_name']] \ + = ( + classified_domain.cert_info.get_cert_status()) provider_access_url = self._get_provider_access_url( util.dict2obj(policy), policy['policy_name'], edge_host_name) @@ -388,7 +424,9 @@ class ServiceController(base.ServiceBase): 'certificate': policy['certificate'] }) ids = policies - return self.responder.updated(json.dumps(ids), links) + return self.responder.updated( + json.dumps(ids), links, + domains_certificate_status=domains_certificate_status) except Exception as e: LOG.exception("Failed to Update Service - {0}". @@ -967,9 +1005,15 @@ class ServiceController(base.ServiceBase): elif domain_obj.certificate == 'san': if edge_host_name is None: raise ValueError("No EdgeHost name provided for SAN Cert") - provider_access_url = '.'.join( - [edge_host_name, - self.driver.akamai_https_access_url_suffix]) + # ugly fix for existing san cert domains, but we will + # have to take it for now + elif edge_host_name.endswith( + self.driver.akamai_https_access_url_suffix): + provider_access_url = edge_host_name + else: + provider_access_url = '.'.join( + [edge_host_name, + self.driver.akamai_https_access_url_suffix]) elif domain_obj.certificate == 'custom': provider_access_url = '.'.join( [dp, self.driver.akamai_https_access_url_suffix]) diff --git a/poppy/provider/base/responder.py b/poppy/provider/base/responder.py index 47cf3232..f2b0812c 100644 --- a/poppy/provider/base/responder.py +++ b/poppy/provider/base/responder.py @@ -65,16 +65,19 @@ class Responder(object): self.provider: provider_response } - def updated(self, provider_service_id, links): + def updated(self, provider_service_id, links, **extras): """updated. :param provider_service_id + :param links + :param **extras :returns provider msg{provider service id} """ provider_response = { "id": provider_service_id, "links": links } + provider_response.update(extras) return { self.provider: provider_response diff --git a/poppy/storage/cassandra/services.py b/poppy/storage/cassandra/services.py index 60571d97..5b711ca1 100644 --- a/poppy/storage/cassandra/services.py +++ b/poppy/storage/cassandra/services.py @@ -31,6 +31,7 @@ from poppy.model.helpers import restriction from poppy.model.helpers import rule from poppy.model import log_delivery as ld from poppy.model import service +from poppy.model import ssl_certificate from poppy.openstack.common import log as logging from poppy.storage import base @@ -202,7 +203,16 @@ CQL_VERIFY_CERT = ''' domain_name FROM certificate_info WHERE domain_name = %(domain_name)s - ALLOW FILTERING +''' + +CQL_SEARCH_CERT_BY_DOMAIN = ''' + SELECT project_id, + flavor_id, + cert_type, + domain_name, + cert_details + FROM certificate_info + WHERE domain_name = %(domain_name)s ''' CQL_UPDATE_SERVICE = CQL_CREATE_SERVICE @@ -488,6 +498,45 @@ class ServicesController(base.ServicesController): "project_id: {0} set to be {1}".format(project_id, project_limit)) + def get_cert_by_domain(self, domain_name, cert_type, + flavor_id, + project_id): + + LOG.info(("Search for cert on '{0}', type: {1}, flavor_id: {2}, " + "project_id: {3}").format(domain_name, cert_type, flavor_id, + project_id)) + args = { + 'domain_name': domain_name.lower() + } + stmt = query.SimpleStatement( + CQL_SEARCH_CERT_BY_DOMAIN, + consistency_level=self._driver.consistency_level) + results = self.session.execute(stmt, args) + + if results: + for r in results: + r_project_id = str(r.get('project_id')) + r_flavor_id = str(r.get('flavor_id')) + r_cert_type = str(r.get('cert_type')) + r_cert_details = {} + cert_details = r.get('cert_details', {}) + # Need to convert cassandra dict into real dict + # And the value of cert_details is a string dict + for key in cert_details: + r_cert_details[key] = json.loads(cert_details[key]) + if r_project_id == str(project_id) and \ + r_flavor_id == str(flavor_id) and \ + r_cert_type == str(cert_type): + res = ssl_certificate.SSLCertificate(r_flavor_id, + domain_name, + r_cert_type, + r_cert_details) + return res + else: + return None + else: + return None + def create(self, project_id, service_obj): """create. @@ -718,8 +767,8 @@ class ServicesController(base.ServicesController): 'domain_name': cert_obj.domain_name, # when create the cert, cert domain has not been assigned yet # In future we can tweak the logic to assign cert_domain - 'cert_domain': '', - 'cert_details': {} + # 'cert_domain': '', + 'cert_details': cert_obj.cert_details } stmt = query.SimpleStatement( CQL_CREATE_CERT, @@ -824,6 +873,7 @@ class ServicesController(base.ServicesController): provider_details[provider_name].error_message) provider_detail_dict[provider_name] = json.dumps( the_provider_detail_dict) + args = { 'project_id': project_id, 'service_id': uuid.UUID(str(service_id)), diff --git a/poppy/transport/pecan/models/request/domain.py b/poppy/transport/pecan/models/request/domain.py index d4d03069..c2fd47e0 100644 --- a/poppy/transport/pecan/models/request/domain.py +++ b/poppy/transport/pecan/models/request/domain.py @@ -14,10 +14,22 @@ # limitations under the License. from poppy.model.helpers import domain +from poppy.model import ssl_certificate def load_from_json(json_data): domain_name = json_data.get('domain') protocol = json_data.get('protocol', 'http') certification_option = json_data.get('certificate', None) - return domain.Domain(domain_name, protocol, certification_option) + res_d = domain.Domain(domain_name, protocol, certification_option) + # Note(tonytan4ever): + # if the domain is in binding status, set the cert_info object + if json_data.get('cert_info') is not None: + cert_info = ssl_certificate.SSLCertificate( + json_data.get('cert_info').get('flavor_id'), + domain_name, + json_data.get('cert_info').get('cert_type'), + json_data.get('cert_info').get('cert_details', {}) + ) + setattr(res_d, 'cert_info', cert_info) + return res_d diff --git a/poppy/transport/pecan/models/response/service.py b/poppy/transport/pecan/models/response/service.py index 15dda283..0da1e15c 100644 --- a/poppy/transport/pecan/models/response/service.py +++ b/poppy/transport/pecan/models/response/service.py @@ -77,18 +77,6 @@ class Model(collections.OrderedDict): for provider_name in service_obj.provider_details: provider_detail = service_obj.provider_details[provider_name] - # add the access urls - access_urls = provider_detail.access_urls - for access_url in access_urls: - if 'operator_url' in access_url: - self['links'].append(link.Model( - access_url['operator_url'], - 'access_url')) - elif 'log_delivery' in access_url: - self['links'].append(link.Model( - access_url['log_delivery'][0]['publicURL'], - 'log_delivery')) - # add any certificate_status for non shared ssl domains # Note(tonytan4ever): for right now we only consider one provider, # in case of multiple providers we really should consider all @@ -101,6 +89,30 @@ class Model(collections.OrderedDict): get_domain_certificate_status( domain_d['domain'])) + # add the access urls + access_urls = provider_detail.access_urls + for access_url in access_urls: + domain_info = next(d for d in self["domains"] + if d['domain'] == access_url['domain']) + # If the domain's status is not deployed, + # don't show the access url since the domain is not usable yet + if domain_info.get("protocol", "http") == "https": + if (provider_detail. + domains_certificate_status. + get_domain_certificate_status( + domain_d['domain']) in ['create_in_progress', + 'failed']): + continue + + if 'operator_url' in access_url: + self['links'].append(link.Model( + access_url['operator_url'], + 'access_url')) + elif 'log_delivery' in access_url: + self['links'].append(link.Model( + access_url['log_delivery'][0]['publicURL'], + 'log_delivery')) + # add any errors error_message = provider_detail.error_message if error_message: diff --git a/tests/unit/distributed_task/taskflow/test_flows.py b/tests/unit/distributed_task/taskflow/test_flows.py index 325a2564..d8493e88 100644 --- a/tests/unit/distributed_task/taskflow/test_flows.py +++ b/tests/unit/distributed_task/taskflow/test_flows.py @@ -89,6 +89,7 @@ class TestFlowRuns(base.TestCase): def patch_create_flow(self, service_controller, storage_controller, dns_controller): storage_controller.get = mock.Mock() + storage_controller.get.return_value = mock.Mock(domains=[]) storage_controller.update = mock.Mock() storage_controller._driver.close_connection = mock.Mock() service_controller.provider_wrapper.create = mock.Mock() @@ -830,7 +831,6 @@ class TestFlowRuns(base.TestCase): dns_controller, storage_controller, memoized_controllers.task_controllers): - self.patch_service_state_flow(service_controller, storage_controller, dns_controller)