x/poppy
Code Issues Proposed changes

feat: addition of the BlueFlood Client

Browse Source 
- add region mapping to every provider driver
- add logic in analytics manager to call the provider, for which the
  domain is present
- add asynchronous requests supports for blueflood and gather results

REQUEST URL:

GET /v1.0/services/{service_id}/analytics?domain={domain}&metricType=requestCount&startTime=2016-01-22T17:42:08&endTime=2016-01-24T17:42:08

RESPONSE:

{
    "domain": "{domain}",
    "requestCount": {
        "India": [{}],
        "EMEA": [{
            "1453420800000": 47,
            "1453507200000": 31,
            "1453593600000": 2
        }],
        "APAC": [{
            "1453593600000": 2
        }],
        "North America": [{}],
        "South America": [{}],
        "Japan": [{
            "1453593600000": 89
        }]
    },
    "flavor": "{flavor}",
    "provider": "{provider}"
}

Partially Implements: blueprint analytics

Change-Id: I82a594c6ed1b2df93158af2b766dbbf2cd5440df
This commit is contained in:
Sriram Madapusi Vasudevan
2016-01-22 16:30:16 -05:00
parent be02a04ce3
commit 0d9f84d616
41 changed files with 1101 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
poppy/provider/akamai/driver.py
View File

@@ -25,6 +25,7 @@ from stevedore import driver
from poppy.common import decorators
from poppy.provider.akamai import controllers
from poppy.provider.akamai import geo_zone_code_mapping
from poppy.provider.akamai.mod_san_queue import zookeeper_queue
from poppy.provider import base
import uuid
@@ -103,6 +104,11 @@ AKAMAI_OPTIONS = [
cfg.StrOpt(
'group_id',
help='Operator groupID'),
# Metrics related configs
cfg.StrOpt('metrics_resolution',
help='Resolution in seconds for retrieving metrics',
default='86400')
]
AKAMAI_GROUP = 'drivers:provider:akamai'
@@ -132,7 +138,7 @@ class CDNProvider(base.Driver):
str(self.akamai_conf.ccu_api_base_url),
'ccu/v2/queues/default'
])
self.regions = geo_zone_code_mapping.REGIONS
self.http_conf_number = self.akamai_conf.akamai_http_config_number
self.https_shared_conf_number = (
self.akamai_conf.akamai_https_shared_config_number)
@@ -186,6 +192,8 @@ class CDNProvider(base.Driver):
self.mod_san_queue = (
zookeeper_queue.ZookeeperModSanQueue(self._conf))
self.metrics_resolution = self.akamai_conf.metrics_resolution
@decorators.lazy_property(write=False)
def san_info_storage(self):
storage_backend_type = 'poppy.provider.akamai.san_info_storage'

3
poppy/provider/akamai/geo_zone_code_mapping.py
View File

@@ -17,9 +17,10 @@ from oslo_log import log
from poppy.model.helpers import geo_zones
# to use log inside worker, we need to directly use logging
LOG = log.getLogger(__name__)
REGIONS = ['North America', 'South America', 'EMEA', 'Japan', 'India', 'APAC']
REGION_COUNTRY_MAPPING = {
'North America': [
'Antigua and Barbuda',

50
poppy/provider/akamai/services.py
View File

@@ -17,6 +17,11 @@ import datetime
import json
import traceback
try: # pragma: no cover
import six.moves.urllib.parse as parse
except ImportError: # pragma: no cover
import urllib.parse as parse
from oslo_log import log
from poppy.common import decorators
@@ -1009,6 +1014,45 @@ class ServiceController(base.ServiceBase):
id_list.append(dp_obj)
return json.dumps(id_list)
def get_metrics_by_domain(self, project_id, domain_name, **extras):
'''Use Akamai's report API to get the metrics by domain.'''
return []
def get_metrics_by_domain(self, project_id, domain_name, regions,
**extras):
"""Use Akamai's report API to get the metrics by domain."""
formatted_results = dict()
metric_buckets = []
metricType = extras['metricType']
startTime = extras['startTime']
endTime = extras['endTime']
metrics_controller = extras['metrics_controller']
resolution = self.driver.metrics_resolution
if 'httpResponseCode' in metricType:
http_series = metricType.split('_')[1]
for region in regions:
metric_buckets.append('_'.join(['requestCount', domain_name,
region,
http_series]))
else:
for region in regions:
metric_buckets.append('_'.join([metricType, domain_name,
region]))
metrics_results = metrics_controller.read(metric_names=metric_buckets,
from_timestamp=startTime,
to_timestamp=endTime,
resolution=resolution)
formatted_results['domain'] = domain_name
formatted_results[metricType] = dict()
for region in regions:
formatted_results[metricType][region] = []
for metric_name, metrics_response in metrics_results:
unquoted_metric_name = parse.unquote(
metric_name.split('_')[2]
).lower()
if region.lower() == unquoted_metric_name:
formatted_results[metricType][region].append(
metrics_response
)
return formatted_results

6
poppy/provider/base/services.py
View File

@@ -77,10 +77,12 @@ class ServicesControllerBase(controller.ProviderControllerBase):
raise NotImplementedError
@abc.abstractmethod
def get_metrics_by_domain(self, project_id, domain_name, **extras):
def get_metrics_by_domain(self, project_id, domain_name, region, **extras):
"""get analytics metrics by domain from provider
:param service_name
:param project_id
:param domain_name
:param regions
:raises NotImplementedError
"""
raise NotImplementedError

1
poppy/provider/cloudfront/driver.py
View File

@@ -45,6 +45,7 @@ class CDNProvider(base.Driver):
self.cloudfront_client = boto.connect_cloudfront(
aws_access_key_id=self.cloudfront_conf.aws_access_key_id,
aws_secret_access_key=self.cloudfront_conf.aws_secret_access_key)
self.regions = []
def is_alive(self):
"""is_alive.

3
poppy/provider/cloudfront/services.py
View File

@@ -99,7 +99,8 @@ class ServiceController(base.ServiceBase):
def get_provider_service_id(self, service_obj):
return service_obj.name
def get_metrics_by_domain(self, project_id, domain_name, **extras):
def get_metrics_by_domain(self, project_id, domain_name, regions,
**extras):
'''Use CloudFronts's API to get the metrics by domain.'''
return []

1
poppy/provider/fastly/driver.py
View File

@@ -55,6 +55,7 @@ class CDNProvider(base.Driver):
setattr(obj, fastly_scheme, self.fastly_conf.scheme)
self.fastly_client = fastly.connect(self.fastly_conf.apikey)
self.regions = []
def is_alive(self):
"""is_alive.

3
poppy/provider/fastly/services.py
View File

@@ -249,6 +249,7 @@ class ServiceController(base.ServiceBase):
def get_provider_service_id(self, service_obj):
return service_obj.service_id
def get_metrics_by_domain(self, project_id, domain_name, **extras):
def get_metrics_by_domain(self, project_id, domain_name, regions,
**extras):
'''Use Fastly's API to get the metrics by domain.'''
return []

3
poppy/provider/maxcdn/driver.py
View File

@@ -36,7 +36,7 @@ MAXCDN_GROUP = 'drivers:provider:maxcdn'
class CDNProvider(base.Driver):
"""MaxCND Provider."""
"""MaxCDN Provider."""
def __init__(self, conf):
"""Init constructor."""
@@ -49,6 +49,7 @@ class CDNProvider(base.Driver):
self.maxcdn_client = maxcdn.MaxCDN(self.maxcdn_conf.alias,
self.maxcdn_conf.consumer_key,
self.maxcdn_conf.consumer_secret)
self.regions = []
def is_alive(self):
"""is_alive.

3
poppy/provider/maxcdn/services.py
View File

@@ -137,7 +137,8 @@ class ServiceController(base.ServiceBase):
def get_provider_service_id(self, service_obj):
return self._map_service_name(service_obj.name)
def get_metrics_by_domain(self, project_id, domain_name, **extras):
def get_metrics_by_domain(self, project_id, domain_name, region,
**extras):
'''Use MaxCDN's API to get the metrics by domain.'''
return []

1
poppy/provider/mock/driver.py
View File

@@ -28,6 +28,7 @@ class CDNProvider(base.Driver):
def __init__(self, conf):
super(CDNProvider, self).__init__(conf)
self.regions = []
def is_alive(self):
"""is_alive.

3
poppy/provider/mock/services.py
View File

@@ -54,7 +54,8 @@ class ServiceController(base.ServiceBase):
def get_provider_service_id(self, service_obj):
return []
def get_metrics_by_domain(self, project_id, domain_name, **extras):
def get_metrics_by_domain(self, project_id, domain_name, regions,
**extras):
return []
@decorators.lazy_property(write=False)