x
/
poppy
Code Issues Proposed changes

Merge "IP restriction Implements blueprint: ip-restrictions"

This commit is contained in:
Jenkins 2015-07-29 13:51:25 +00:00 committed by Gerrit Code Review
parent 6d0c99c22d d7fe8fdbf9
commit 4071036f8e
17 changed files with 462 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
poppy/model/helpers/rule.py
View File

@ -21,7 +21,7 @@ class Rule(common.DictSerializableModel):
def __init__(self, name=None,
referrer=None, http_host=None, client_ip=None,
http_method=None, request_url="/*"):
geography=None, http_method=None, request_url="/*"):
self._name = name
self._request_url = request_url
@ -33,6 +33,10 @@ class Rule(common.DictSerializableModel):
self._client_ip = client_ip
if http_method:
self._http_method = http_method
if geography:
self._geography = geography
if request_url:
self._request_url = request_url
@property
def name(self):
@ -79,3 +83,12 @@ class Rule(common.DictSerializableModel):
@request_url.setter
def request_url(self, value):
self._request_url = value
@property
def geography(self):
"""http_host."""
return self._geography
@geography.setter
def geography(self, value):
self._geography = value

117
poppy/provider/akamai/services.py
View File

@ -134,7 +134,6 @@ class ServiceController(base.ServiceBase):
def update(self, provider_service_id,
service_obj):
try:
# depending on domains field presented or not, do PUT/POST
# and depending on origins field presented or not, set behavior on
@ -541,28 +540,78 @@ class ServiceController(base.ServiceBase):
'/' + rule_entry.request_url)
# restriction implementation for akamai
# for each restriction rule
# restriction entities include: referrer, geography, client_ip
restriction_entities = ['referrer', 'client_ip']
class entityRequestUrlMappingList(dict):
'''A dictionary with a name attribute'''
def __init__(self, name, orig_dict):
self.name = name
self.update(orig_dict)
# classify restriction/rules based on their white/black-list
white_list_entities = entityRequestUrlMappingList(
'whitelist',
{entity: {} for entity
in restriction_entities})
black_list_entities = entityRequestUrlMappingList(
'blacklist',
{entity: {} for entity
in restriction_entities})
for restriction_rule in restriction_rules:
entity_rule_mapping = {}
if restriction_rule.type == 'whitelist':
entity_rule_mapping = white_list_entities
elif restriction_rule.type == 'blacklist':
entity_rule_mapping = black_list_entities
for rule_entry in restriction_rule.rules:
if rule_entry.referrer is not None:
# classify rules based on their entities, then request_urls
if getattr(rule_entry, "referrer", None) is not None:
if (rule_entry.request_url not in
entity_rule_mapping['referrer']):
entity_rule_mapping['referrer'][rule_entry.request_url]\
= [rule_entry]
else:
entity_rule_mapping['referrer'][rule_entry.request_url]\
.append(rule_entry)
elif getattr(rule_entry, "client_ip", None) is not None:
if (rule_entry.request_url not in
entity_rule_mapping['client_ip']):
entity_rule_mapping['client_ip'][rule_entry.request_url]\
= [rule_entry]
else:
entity_rule_mapping['client_ip'][rule_entry.request_url]\
.append(rule_entry)
for entity_request_url_rule_mapping in [white_list_entities,
black_list_entities]:
for entity in entity_request_url_rule_mapping:
for request_url in entity_request_url_rule_mapping[entity]:
found_match = False
referrer_whitelist_value = ' '.join(
['*%s*' % referrer
for referrer
in rule_entry.referrer.split(' ')
])
# need to write up a function gets the value of behavior
behavior_name = self._get_behavior_name(
entity, entity_request_url_rule_mapping.name)
behavior_value = self._get_behavior_value(
entity,
entity_request_url_rule_mapping[entity][request_url])
behavior_dict = {
'name': behavior_name,
'value': behavior_value
}
# if we have a matches rule already
for rule in rules_list:
for match in rule['matches']:
if rule_entry.request_url == match['value']:
if request_url == match['value']:
# we found an existing matching rule.
# add the cache behavior to it
# add the whitelist/blacklist behavior to it
found_match = True
rule['behaviors'].append({
'name': 'referer-whitelist',
'value': referrer_whitelist_value
})
rule['behaviors'].append(behavior_dict)
# if there is no matches entry yet for this rule
if not found_match:
@ -576,17 +625,45 @@ class ServiceController(base.ServiceBase):
if rule_entry.request_url is not None:
match_rule = {
'name': 'url-wildcard',
'value': rule_entry.request_url
'value': request_url
}
rule_dict_template['matches'].append(match_rule)
rule_dict_template['behaviors'].append({
'name': 'referer-whitelist',
'value': referrer_whitelist_value
})
rule_dict_template['behaviors'].append(
behavior_dict)
rules_list.append(rule_dict_template)
# end loop - restriction_rule.rules
# end lop - restriction_rules
# end loop - request url
# end loop - entity
def _get_behavior_name(self, entity, entity_restriction_type):
prefix = suffix = None
if entity == 'referrer':
prefix = 'referer'
elif entity == 'client_ip':
prefix = 'ip'
if entity_restriction_type == 'whitelist':
suffix = 'whitelist'
elif entity_restriction_type == 'blacklist':
suffix = 'blacklist'
return '-'.join([prefix, suffix])
def _get_behavior_value(self, entity, rule_entries):
if entity == 'referrer':
return ' '.join(
['*%s*' % referrer
for rule_entry
in rule_entries
for referrer
in rule_entry.referrer.split()
])
elif entity == 'client_ip':
return ' '.join(
['%s' % rule_entry.client_ip
for rule_entry
in rule_entries
])
def _process_caching_rules(self, caching_rules, rules_list):
# akamai requires all caching rules to start with '/'

3
poppy/storage/cassandra/services.py
View File

@ -640,8 +640,11 @@ class ServicesController(base.ServicesController):
restrictions = [restriction.Restriction(
r.get('name'),
r.get('type'),
[rule.Rule(r_rule.get('name'),
referrer=r_rule.get('referrer'),
client_ip=r_rule.get('client_ip'),
geography=r_rule.get('geography'),
request_url=r_rule.get('request_url', "/*") or "/*")
for r_rule in r['rules']])
for r in restrictions]

1
poppy/transport/pecan/models/request/rule.py
View File

@ -24,4 +24,5 @@ def load_from_json(json_data):
res.http_method = json_data.get('http_method', None)
res.client_ip = json_data.get('client_ip', None)
res.request_url = json_data.get('request_url', None)
res.geography = json_data.get('geography', None)
return res

3
poppy/transport/pecan/models/response/rule.py
View File

@ -29,7 +29,8 @@ class Model(collections.OrderedDict):
self['name'] = util.help_escape(rule.name)
self['request_url'] = util.help_escape(rule.request_url)
for attr_name in ['http_host', 'http_method',
'client_ip', 'referrer']:
'client_ip', 'request_url',
'referrer', 'geography']:
attr = getattr(rule, attr_name, None)
if attr is not None:
self[attr_name] = util.help_escape(attr)

17
poppy/transport/validators/helpers.py
View File

@ -224,19 +224,7 @@ def is_valid_service_configuration(service, schema):
else:
domains.append(domain_value)
# 4. referrer restriction paths must be unique
if 'restrictions' in service:
restriction_paths = []
for restriction in service['restrictions']:
if 'rules' in restriction:
for rule in restriction['rules']:
if 'referrer' in rule:
request_url = rule.get('request_url', '/*')
if request_url in restriction_paths:
raise exceptions.ValidationFailed(
'Referrer - the request_url must be unique')
else:
restriction_paths.append(request_url)
# We allow multiple restrictions rules on the same path
# 5. domains protocols must be of the same type, and domains protocols must
# match the description (ssl/port) of the origin
@ -312,9 +300,10 @@ def is_valid_service_configuration(service, schema):
}
blacklist_restriction_entities = {
}
if 'restrictions' in service:
for restriction in service['restrictions']:
if restriction.get('type', 'whitelist') == 'blacklist':
if restriction.get('type', 'blacklist') == 'blacklist':
for rule in restriction['rules']:
entity = None
request_url = '/*'

71
poppy/transport/validators/schemas/service.py
View File

@ -432,17 +432,66 @@ class ServiceSchema(schema_base.SchemaBase):
'type': 'array',
'required': True,
'items': {
'type': 'object',
'properties': {
'name': {
'type': 'string',
'minLength': 1,
'maxLength': 256},
'referrer': {
'type': 'string',
'minLength': 3,
'maxLength': 1024}
}},
'type': [
{
'type': 'object',
"properties": {
"name": {
"type": "string",
},
"referrer": {
"type": "string",
'required': True,
},
'request_url': {
'type': 'string',
'minLength': 2,
'maxLength': 100
}
},
"additionalProperties": False,
}, {
'type': 'object',
"properties": {
"name": {
"type": "string",
},
"geography": {
"type": "string",
'required': True,
},
'request_url': {
'type': 'string',
'minLength': 2,
'maxLength': 100
}
},
"additionalProperties": False
}, {
'type': 'object',
"properties": {
"name": {
"type": "string",
},
"client_ip": {
"type": "string",
'pattern': re.compile(
"^\d{1,3}\.\d{1,3}"
"\.\d{1,3}"
"\.\d{1,3}$"
),
'required': True,
},
'request_url': {
'type': 'string',
'minLength': 2,
'maxLength': 100
}
},
"additionalProperties": False
}
]
}
}},
},
},

109
tests/api/services/data_create_service.json
View File

@ -38,12 +38,19 @@
}
],
"restrictions_list": [
{"name": "test",
{
"name": "test",
"type": "whitelist",
"rules": [
{
"name": "only me",
"referrer": "www.mywebsite.com",
"request_url" : "/*"
},
{
"name": "only 1234",
"client_ip": "1.2.3.4",
"request_url" : "/*"
}
]}
],
@ -435,6 +442,104 @@
}
]}
]
}
},
"ip_restriction": {
"name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
"origin_list": [{
"origin": "mywebsite1.com",
"port": 80,
"ssl": false,
"rules": [
{
"name" : "default",
"request_url" : "/*"
}]
}],
"caching_list": [
{
"name": "default",
"ttl": 3600,
"rules": [
{
"name" : "default",
"request_url" : "/*"
}
]
},
{
"name": "home",
"ttl": 1200,
"rules": [
{
"name" : "index",
"request_url" : "/index.htm"
}
]
}
],
"restrictions_list": [
{
"name": "test",
"type": "whitelist",
"rules": [
{
"name": "only 1234",
"client_ip": "1.2.3.4",
"request_url" : "/*"
}
]}
]
},
"ip_restriction_blacklist": {
"name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
"origin_list": [{
"origin": "mywebsite1.com",
"port": 80,
"ssl": false,
"rules": [
{
"name" : "default",
"request_url" : "/*"
}]
}],
"caching_list": [
{
"name": "default",
"ttl": 3600,
"rules": [
{
"name" : "default",
"request_url" : "/*"
}
]
},
{
"name": "home",
"ttl": 1200,
"rules": [
{
"name" : "index",
"request_url" : "/index.htm"
}
]
}
],
"restrictions_list": [
{
"name": "test",
"type": "blacklist",
"rules": [
{
"name": "only 1234",
"client_ip": "1.2.3.4",
"request_url" : "/*"
}
]}
]
}
}

37
tests/api/services/data_create_service_negative.json
View File

@ -121,6 +121,7 @@
"request_url" : "/index.htm"}]}],
"restrictions_list": [
{"name": "test",
"type": "whitelist",
"rules": [{"name": "only me",
"referrer": "www.mywebsite.com"}]}
]
@ -459,6 +460,21 @@
"referrer": "www.mywebsite.com"}]}
]
},
"restrictions_no_name": {
"service_name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
"origin_list": [{"origin": "mywebsite1.com",
"port": 443,
"ssl": false}],
"caching_list": [{"name": "default", "ttl": 3600},
{"name": "home",
"ttl": 3600,
"rules": [{"name" : "index",
"request_url" : "/index.htm"}]}],
"restrictions_list": [{
"rules": [{"referrer": "www.abc.com"}]}]
},
"restrictions_no_rules": {
"service_name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
@ -473,7 +489,7 @@
"request_url" : "/index.htm"}]}],
"restrictions_list": [{"name":"default"}]
},
"restrictions_rule_no_name": {
"restrictions_invalid_ip_address": {
"service_name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
@ -486,7 +502,24 @@
"rules": [{"name" : "index",
"request_url" : "/index.htm"}]}],
"restrictions_list": [{"name":"default",
"rules": [{}]}]
"rules": [{"client_ip": "1.2.3"}]}]
},
"restrictions_black_and_whitelist_ip": {
"service_name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
"origin_list": [{"origin": "mywebsite1.com",
"port": 443,
"ssl": false}],
"caching_list": [{"name": "default", "ttl": 3600},
{"name": "home",
"ttl": 3600,
"rules": [{"name" : "index",
"request_url" : "/index.htm"}]}],
"restrictions_list": [{"name":"default",
"rules": [{"client_ip": "1.2.3.4"}]},{
"name":"default-blacklist", "type": "blacklist",
"rules": [{"client_ip": "5.6.7.8"}]}]
},
"origin_invalid_port": {
"service_name": "origin_invalid_port",

50
tests/api/services/data_patch_service.json
View File

@ -198,6 +198,7 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "www.thesite.com", "request_url": "/images/*"}]
}}
],
@ -205,11 +206,13 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "www.thesite.com", "request_url": "/images2/*"}]
}},
{"op": "replace",
"path": "/restrictions/0",
"value": {"name": "new_restricted",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "www.thesite.com", "request_url": "/*"}]
}}
],
@ -233,6 +236,7 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "r",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "www.thesite.com", "request_url": "/images/*"}]
}
}
@ -276,6 +280,7 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted",
"type": "whitelist",
"rules": [{"name": "r", "referrer": "www.thesite.com", "request_url": "/images/*"}]
}
}
@ -284,6 +289,7 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "a.co", "request_url":"/images/*"}]
}
}
@ -309,5 +315,47 @@
"value": {"origin": "1.1.1.1", "port": 443, "ssl": true,
"rules": [{"name" : "origin", "request_url" : "/origin2.htm"}], "hostheadertype": "custom", "hostheadervalue": "www.customweb.com"}}
],
"empty_list": []
"empty_list": [],
"replace_change_restricted_ip_to_another":[
{"op": "replace",
"path": "/restrictions/0",
"value": {"name": "website only",
"type": "whitelist",
"rules": [{"name": "mywebsite.com",
"referrer": "www.mywebsite.com",
"request_url": "/*"
}, {"name": "myrestricted-ip",
"client_ip": "5.6.7.8",
"request_url": "/*"}
]}}
],
"replace_remove_all_restrictions":[
{"op": "remove",
"path": "/restrictions/0"}
],
"replace_remove_restricted_ip":[
{"op": "replace",
"path": "/restrictions/0",
"value": {"name": "website only",
"type": "whitelist",
"rules": [{"name": "mywebsite.com",
"referrer": "www.mywebsite.com",
"request_url": "/*"
}]}}
],
"replace_add_new_restricted_ip":[
{"op": "replace",
"path": "/restrictions/0",
"value": {"name": "website only",
"type": "whitelist",
"rules": [{"name": "mywebsite.com",
"referrer": "www.mywebsite.com",
"request_url": "/*"
}, {"name": "myrestricted-ip",
"client_ip": "1.2.3.4",
"request_url": "/*"}
,{"name": "myrestricted-ip-2",
"client_ip": "5.6.7.8",
"request_url": "/*"}]}}
]
}

26
tests/api/services/data_patch_service_negative.json
View File

@ -203,5 +203,31 @@
"path": "/domains/-",
"value": {"domain": "replaceme.com", "protocol": "https",
"certificate": "shared"}}
],
"add_restrictions_with_invalid_ip": [
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted-ip2",
"rules": [{"name": "rule1", "client_ip": "13.4.5"}]
}
}
],
"add_restrictions_with_conflict_blacklist_referrer": [
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted-blacklist",
"type": "blacklist",
"rules": [{"name": "rule1", "referrer": "myblacklist.com"}]
}
}
],
"add_restrictions_with_conflict_blacklist": [
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted-blacklist",
"type": "blacklist",
"rules": [{"name": "rule1", "client_ip": "5.6.7.8"}]
}
}
]
}

6
tests/functional/transport/pecan/controllers/data_create_service.json
View File

@ -26,7 +26,11 @@
"rules": [
{
"name": "mocksite.com",
"http_host": "www.mocksite.com"
"referrer": "www.mocksite.com"
},
{
"name": "us-website-only",
"geography": "USA"
}
]
}

2
tests/functional/transport/pecan/controllers/data_create_service_duplicate.json
View File

@ -26,7 +26,7 @@
"rules": [
{
"name": "mocksite.com",
"http_host": "www.mocksite.com"
"referrer": "www.mocksite.com"
}
]
}

4
tests/functional/transport/pecan/controllers/test_services.py
View File

@ -96,7 +96,7 @@ class ServiceControllerTest(base.FunctionalTest):
"rules": [
{
"name": "mocksite.com",
"http_host": "www.mocksite.com"
"referrer": "www.mocksite.com"
}
]
}
@ -461,7 +461,7 @@ class ServiceControllerTest1(base.FunctionalTest):
"rules": [
{
"name": "mocksite.com",
"http_host": "www.mocksite.com"
"referrer": "www.mocksite.com"
}
]
}

5
tests/unit/model/helpers/test_rule.py
View File

@ -30,6 +30,7 @@ class TestRule(base.TestCase):
client_ip = '192.168.1.1'
http_method = 'POST'
request_url = '/index.html'
geography = 'USA'
myrule = rule.Rule(name)
@ -53,3 +54,7 @@ class TestRule(base.TestCase):
# request_url
myrule.request_url = request_url
self.assertEqual(myrule.request_url, request_url)
# geography
myrule.geography = geography
self.assertEqual(myrule.geography, geography)

46
tests/unit/provider/akamai/data_service.json
View File

@ -61,6 +61,7 @@
"restrictions": [
{
"name": "website only",
"type": "whitelist",
"rules": [
{
"name": "mocksite.com",
@ -69,11 +70,16 @@
]
},
{
"name": "graphic only",
"name": "graphic and US only",
"type": "whitelist",
"rules": [
{
"name": "mockgraphicsite.com",
"referrer": "www.mocksitegraphic.com"
},
{
"name": "US geo restriction",
"geography": "USA"
}
]
}
@ -221,5 +227,43 @@
}
],
"flavor_id" : "standard"
},
"ip_restrictions_whitelist": {
"name" : "mysite.com",
"domains": [
{"domain": "parsely.sage.com"}
],
"origins": [
{"origin": "mockdomain-text.com", "ssl": false, "port": 80,
"rules": [{"name": "global", "request_url": "/*"},
{"name": "text", "request_url": "/text"}]}
],
"restrictions": [
{
"name": "only from ip 1234",
"type": "blacklist",
"rules": [
{
"name": "only ip",
"client_ip": "1.2.3.4"
}
]
}
],
"caching": [
{"name": "default", "ttl": 1200 },
{"name": "img-only",
"ttl": 1800,
"rules": [
{ "name": "jpeg-rules",
"request_url": "/*.jpeg"
},
{ "name": "gif-rules",
"request_url": "/*.gif"
}
]
}
],
"flavor_id" : "standard"
}
}

8
tests/unit/provider/akamai/data_update_service.json
View File

@ -44,6 +44,7 @@
"restrictions": [
{
"name": "website only",
"type": "whitelist",
"rules": [
{
"name": "mocksite.com",
@ -52,11 +53,16 @@
]
},
{
"name": "graphic only",
"name": "graphic and US only",
"type": "whitelist",
"rules": [
{
"name": "mockgraphicsite.com",
"referrer": "www.mocksitegraphic.com"
},
{
"name": "US geo restriction",
"geography": "USA"
}
]
}