x/quark
Code Issues Proposed changes

Ip notifications (#558)

Browse Source 
* * additional ip billing notifications

JIRA:NCP-1871

* * made public_net_id a config option; fixed the --notify option
This commit is contained in:
Alexander Medvedev
2016-06-06 12:07:22 -05:00
committed by Justin Hammond
parent 366af43984
commit 5ad02e0950
11 changed files with 737 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

297
quark/billing.py Normal file
View File

@@ -0,0 +1,297 @@
# Copyright 2016 Openstack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Calculations for different cases for additional IP billing
A - allocated
D - deallocated
| - start or end of the billing period, typically 24 hrs
A | | D
| A | D
A | D |
| A D |
---IP CURRENTLY ALLOCATED=has tenant_id (send usage at end of period)---
case 1 (allocated before period began, deallocated after period ended)
send usage as 24 hours
case 2 (allocated during period, deallocated after period ended)
send usage as timedelta(period_end - allocated_at)
---IP DEALLOCATED DURING PERIOD (send usage at deallocation time)---
case 3 (allocated before period began, deallocated during period)
send usage as timedelta(deallocated_at - period_start)
case 4 (allocated during period, deallocated during period)
send usage as timedelta(deallocated_at - allocated_at)
NOTE: notifications are issued at these times:
case1 and case2 - this script runs nightly and processes the entries
case3 and case4 - the notifications are sent immediately when the ip
is deallocated
NOTE: assumes that the beginning of a billing cycle is midnight.
"""
import datetime
from neutron.common import rpc as n_rpc
from oslo_log import log as logging
from sqlalchemy import and_, or_, null
from quark.db import models
from quark import network_strategy
LOG = logging.getLogger(__name__)
PUBLIC_NETWORK_ID = network_strategy.STRATEGY.get_public_net_id()
# NOTE: this will most likely go away to be done in yagi
EVENT_TYPE_2_CLOUDFEEDS = {
'ip.exists': 'USAGE',
'ip.add': 'CREATE',
'ip.delete': 'DELETE',
'ip.associate': 'UP',
'ip.disassociate': 'DOWN'
}
def do_notify(context, event_type, payload):
"""Generic Notifier.
Parameters:
- `context`: session context
- `event_type`: the event type to report, i.e. ip.usage
- `payload`: dict containing the payload to send
"""
LOG.debug('IP_BILL: notifying {}'.format(payload))
notifier = n_rpc.get_notifier('network')
notifier.info(context, event_type, payload)
def notify(context, event_type, ipaddress, send_usage=False):
"""Method to send notifications.
We must send USAGE when a public IPv4 address is deallocated or a FLIP is
associated.
Parameters:
- `context`: the context for notifier
- `event_type`: the event type for IP allocate, deallocate, associate,
disassociate
- `ipaddress`: the ipaddress object to notify about
Returns:
nothing
Notes: this may live in the billing module
"""
# ip.add needs the allocated_at time.
# All other events need the current time.
ts = ipaddress.allocated_at if event_type == 'ip.add' else _now()
payload = build_payload(ipaddress, event_type, event_time=ts)
# Send the notification with the payload
do_notify(context, event_type, payload)
# When we deallocate an IP or associate a FLIP we must send
# a usage message to billing.
# In other words when we supply end_time we must send USAGE to billing
# immediately.
# Our billing period is 24 hrs. If the address was allocated after midnight
# send the start_time as as. If the address was allocated yesterday, then
# send midnight as the start_time.
# Note: if allocated_at is empty we assume today's midnight.
if send_usage:
if ipaddress.allocated_at is not None and \
ipaddress.allocated_at >= _midnight_today():
start_time = ipaddress.allocated_at
else:
start_time = _midnight_today()
payload = build_payload(ipaddress,
'ip.exists',
start_time=start_time,
end_time=ts)
do_notify(context, 'ip.exists', payload)
def build_payload(ipaddress,
event_type,
event_time=None,
start_time=None,
end_time=None):
"""Method builds a payload out of the passed arguments.
Parameters:
`ipaddress`: the models.IPAddress object
`event_type`: USAGE,CREATE,DELETE,SUSPEND,or UNSUSPEND
`start_time`: startTime for cloudfeeds
`end_time`: endTime for cloudfeeds
Returns a dictionary suitable to notify billing.
Message types mapping to cloud feeds for references:
ip.exists - USAGE
ip.add - CREATE
ip.delete - DELETE
ip.associate - UP
ip.disassociate - DOWN
Refer to: http://rax.io/cf-api for more details.
"""
# This is the common part of all message types
payload = {
'event_type': unicode(EVENT_TYPE_2_CLOUDFEEDS[event_type]),
'tenant_id': unicode(ipaddress.used_by_tenant_id),
'ip_address': unicode(ipaddress.address_readable),
'subnet_id': unicode(ipaddress.subnet_id),
'network_id': unicode(ipaddress.network_id),
'public': True if ipaddress.network_id == PUBLIC_NETWORK_ID else False,
'ip_version': int(ipaddress.version),
'ip_type': unicode(ipaddress.address_type),
'id': unicode(ipaddress.id)
}
# Depending on the message type add the appropriate fields
if event_type == 'ip.exists':
if start_time is None or end_time is None:
raise ValueError('IP_BILL: {} start_time/end_time cannot be empty'
.format(event_type))
payload.update({
'startTime': unicode(convert_timestamp(start_time)),
'endTime': unicode(convert_timestamp(end_time))
})
elif event_type == 'ip.add':
if event_time is None:
raise ValueError('IP_BILL: {}: event_time cannot be NULL'
.format(event_type))
payload.update({
'eventTime': unicode(convert_timestamp(event_time)),
})
elif event_type == 'ip.delete':
if event_time is None:
raise ValueError('IP_BILL: {}: event_time cannot be NULL'
.format(event_type))
payload.update({
'eventTime': unicode(convert_timestamp(event_time))
})
elif event_type == 'ip.associate' or event_type == 'ip.disassociate':
if event_time is None:
raise ValueError('IP_BILL: {}: event_time cannot be NULL'
.format(event_type))
# only pass floating ip addresses through this
if ipaddress.address_type not in ['floating', 'scaling']:
raise ValueError('IP_BILL: {} only valid for floating IPs'.
format(event_type),
' got {} instead'.format(ipaddress.address_type))
payload.update({'eventTime': unicode(convert_timestamp(event_time))})
else:
raise ValueError('IP_BILL: bad event_type: {}'.format(event_type))
return payload
def build_full_day_ips(query, period_start, period_end):
"""Method to build an IP list for the case 1
when the IP was allocated before the period start
and is still allocated after the period end.
This method only looks at public IPv4 addresses.
"""
# Filter out only IPv4 that have not been deallocated
ip_list = query.\
filter(models.IPAddress.version == 4L).\
filter(models.IPAddress.network_id == PUBLIC_NETWORK_ID).\
filter(models.IPAddress.used_by_tenant_id is not None).\
filter(models.IPAddress.allocated_at != null()).\
filter(models.IPAddress.allocated_at < period_start).\
filter(or_(models.IPAddress._deallocated is False,
models.IPAddress.deallocated_at == null(),
models.IPAddress.deallocated_at >= period_end)).all()
return ip_list
def build_partial_day_ips(query, period_start, period_end):
"""Method to build an IP list for the case 2
when the IP was allocated after the period start and
is still allocated after the period end.
This method only looks at public IPv4 addresses.
"""
# Filter out only IPv4 that were allocated after the period start
# and have not been deallocated before the period end.
# allocated_at will be set to a date
ip_list = query.\
filter(models.IPAddress.version == 4L).\
filter(models.IPAddress.network_id == PUBLIC_NETWORK_ID).\
filter(models.IPAddress.used_by_tenant_id is not None).\
filter(and_(models.IPAddress.allocated_at != null(),
models.IPAddress.allocated_at >= period_start,
models.IPAddress.allocated_at < period_end)).\
filter(or_(models.IPAddress._deallocated is False,
models.IPAddress.deallocated_at == null(),
models.IPAddress.deallocated_at >= period_end)).all()
return ip_list
def calc_periods(hour=0, minute=0):
"""Returns a tuple of start_period and end_period.
Assumes that the period is 24-hrs.
Parameters:
- `hour`: the hour from 0 to 23 when the period ends
- `minute`: the minute from 0 to 59 when the period ends
This method will calculate the end of the period as the closest hour/minute
going backwards.
It will also calculate the start of the period as the passed hour/minute
but 24 hrs ago.
Example, if we pass 0, 0 - we will get the events from 0:00 midnight of the
day before yesterday until today's midnight.
If we pass 2,0 - we will get the start time as 2am of the previous morning
till 2am of today's morning.
By default it's midnight.
"""
# Calculate the time intervals in a usable form
period_end = datetime.datetime.utcnow().replace(hour=hour,
minute=minute,
second=0,
microsecond=0)
period_start = period_end - datetime.timedelta(days=1)
# period end should be slightly before the midnight.
# hence, we subtract a second
# this will force period_end to store something like:
# datetime.datetime(2016, 5, 19, 23, 59, 59, 999999)
# instead of:
# datetime.datetime(2016, 5, 20, 0, 0, 0, 0)
period_end -= datetime.timedelta(seconds=1)
return (period_start, period_end)
def _midnight_today():
return datetime.datetime.utcnow().replace(hour=0,
minute=0,
second=0)
def convert_timestamp(ts):
"""Converts the timestamp to a format suitable for Billing.
Examples of a good timestamp for startTime, endTime, and eventTime:
'2016-05-20T00:00:00Z'
Note the trailing 'Z'. Python does not add the 'Z' so we tack it on
ourselves.
"""
return ts.isoformat() + 'Z'
def _now():
"""Method to get the utcnow without microseconds"""
return datetime.datetime.utcnow().replace(microsecond=0)

38
quark/ipam.py
View File

@@ -25,7 +25,6 @@ import uuid
import netaddr
from neutron.common import exceptions as n_exc_ext
from neutron.common import rpc as n_rpc
from neutron_lib import exceptions as n_exc
from oslo_concurrency import lockutils
from oslo_config import cfg
@@ -33,6 +32,7 @@ from oslo_db import exception as db_exception
from oslo_log import log as logging
from oslo_utils import timeutils
from quark.billing import notify
from quark.db import api as db_api
from quark.db import ip_types
from quark.db import models
@@ -480,6 +480,9 @@ class QuarkIpam(object):
port_id=port_id,
address_type=kwargs.get('address_type', ip_types.FIXED))
address["deallocated"] = 0
# alexm: instead of notifying billing from here we notify from
# allocate_ip_address() when it's clear that the IP
# allocation was successful
except db_exception.DBDuplicateEntry:
raise q_exc.CannotAllocateReallocateableIP(ip_address=next_ip)
except db_exception.DBError:
@@ -547,12 +550,16 @@ class QuarkIpam(object):
try:
with context.session.begin():
return db_api.ip_address_create(
address = db_api.ip_address_create(
context, address=ip_address,
subnet_id=subnet["id"],
version=subnet["ip_version"], network_id=net_id,
address_type=kwargs.get('address_type',
ip_types.FIXED))
# alexm: need to notify from here because this code
# does not go through the _allocate_from_subnet() path.
notify(context, 'ip.add', address)
return address
except db_exception.DBDuplicateEntry:
# This shouldn't ever happen, since we hold a unique MAC
# address from the previous IPAM step.
@@ -600,17 +607,6 @@ class QuarkIpam(object):
return new_addresses
def _notify_new_addresses(self, context, new_addresses):
for addr in new_addresses:
payload = dict(used_by_tenant_id=addr["used_by_tenant_id"],
ip_block_id=addr["subnet_id"],
ip_address=addr["address_readable"],
device_ids=[p["device_id"] for p in addr["ports"]],
created_at=addr["created_at"])
n_rpc.get_notifier("network").info(context,
"ip_block.address.create",
payload)
@ipam_logged
def allocate_ip_address(self, context, new_addresses, net_id, port_id,
reuse_after, segment_id=None, version=None,
@@ -703,7 +699,9 @@ class QuarkIpam(object):
_try_allocate_ip_address(ipam_log)
if self.is_strategy_satisfied(new_addresses, allocate_complete=True):
self._notify_new_addresses(context, new_addresses)
# Only notify when all went well
for address in new_addresses:
notify(context, 'ip.add', address)
LOG.info("IPAM for port ID {0} completed with addresses "
"{1}".format(port_id,
[a["address_readable"]
@@ -720,15 +718,7 @@ class QuarkIpam(object):
address["deallocated"] = 1
address["address_type"] = None
payload = dict(used_by_tenant_id=address["used_by_tenant_id"],
ip_block_id=address["subnet_id"],
ip_address=address["address_readable"],
device_ids=[p["device_id"] for p in address["ports"]],
created_at=address["created_at"],
deleted_at=timeutils.utcnow())
n_rpc.get_notifier("network").info(context,
"ip_block.address.delete",
payload)
notify(context, 'ip.delete', address, send_usage=True)
def deallocate_ips_by_port(self, context, port=None, **kwargs):
ips_to_remove = []
@@ -778,6 +768,7 @@ class QuarkIpam(object):
# SQLAlchemy caching.
context.session.add(flip)
context.session.flush()
notify(context, 'ip.disassociate', flip)
driver = registry.DRIVER_REGISTRY.get_driver()
driver.remove_floating_ip(flip)
elif len(flip.fixed_ips) > 1:
@@ -791,6 +782,7 @@ class QuarkIpam(object):
context, flip, fix_ip)
context.session.add(flip)
context.session.flush()
notify(context, 'ip.disassociate', flip)
else:
remaining_fixed_ips.append(fix_ip)
port_fixed_ips = {}

11
quark/network_strategy.py
View File

@@ -23,7 +23,9 @@ CONF = cfg.CONF
quark_opts = [
cfg.StrOpt('default_net_strategy', default='{}',
help=_("Default network assignment strategy"))
help=_("Default network assignment strategy")),
cfg.StrOpt('public_net_id', default='00000000-0000-0000-0000-000000000000',
help=_("Public network id"))
]
CONF.register_opts(quark_opts, "QUARK")
@@ -98,5 +100,12 @@ class JSONStrategy(object):
return None
return self.subnet_strategy.get(subnet_id)["network_id"]
def get_public_net_id(self):
"""Returns the public net id"""
for id, net_params in self.strategy.iteritems():
if id == CONF.QUARK.public_net_id:
return id
return None
STRATEGY = JSONStrategy()

23
quark/plugin_modules/floating_ips.py
View File

@@ -17,6 +17,7 @@ from neutron_lib import exceptions as n_exc
from oslo_config import cfg
from oslo_log import log as logging
from quark import billing
from quark.db import api as db_api
from quark.db import ip_types
from quark.drivers import floating_ip_registry as registry
@@ -164,6 +165,9 @@ def _create_flip(context, flip, port_fixed_ips):
context.session.rollback()
raise
# alexm: Notify from this method for consistency with _delete_flip
billing.notify(context, 'ip.associate', flip)
def _get_flip_fixed_ip_by_port_id(flip, port_id):
for fixed_ip in flip.fixed_ips:
@@ -181,6 +185,13 @@ def _update_flip(context, flip_id, ip_type, requested_ports):
{"port_id": "<id of port>", "fixed_ip": "<fixed ip address>"}
:return: quark.models.IPAddress
"""
# This list will hold flips that require notifications.
# Using sets to avoid dups, if any.
notifications = {
'ip.associate': set(),
'ip.disassociate': set()
}
context.session.begin()
try:
flip = db_api.floating_ip_find(context, id=flip_id, scope=db_api.ONE)
@@ -222,6 +233,7 @@ def _update_flip(context, flip_id, ip_type, requested_ports):
for port_id in removed_port_ids:
port = db_api.port_find(context, id=port_id, scope=db_api.ONE)
flip = db_api.port_disassociate_ip(context, [port], flip)
notifications['ip.disassociate'].add(flip)
fixed_ip = _get_flip_fixed_ip_by_port_id(flip, port_id)
if fixed_ip:
flip = db_api.floating_ip_disassociate_fixed_ip(
@@ -245,6 +257,7 @@ def _update_flip(context, flip_id, ip_type, requested_ports):
raise q_exc.NoAvailableFixedIpsForPort(port_id=port_id)
port_fixed_ips[port_id] = {'port': port, 'fixed_ip': fixed_ip}
flip = db_api.port_associate_ip(context, [port], flip, [port_id])
notifications['ip.associate'].add(flip)
flip = db_api.floating_ip_associate_fixed_ip(context, flip,
fixed_ip)
@@ -264,6 +277,12 @@ def _update_flip(context, flip_id, ip_type, requested_ports):
except Exception:
context.session.rollback()
raise
# Send notifications for possible associate/disassociate events
for notif_type, flip_set in notifications.iteritems():
for flip in flip_set:
billing.notify(context, notif_type, flip)
# NOTE(blogan): ORM does not seem to update the model to the real state
# of the database, so I'm doing an explicit refresh for now.
context.session.refresh(flip)
@@ -309,6 +328,10 @@ def _delete_flip(context, id, address_type):
'may need to be handled manually in the unicorn API. '
'Error: %s' % e.message)
# alexm: Notify from this method because we don't have the flip object
# in the callers
billing.notify(context, 'ip.disassociate', flip)
def create_floatingip(context, content):
"""Allocate or reallocate a floating IP.

7
quark/tests/functional/plugin_modules/test_floating_ips.py
View File

@@ -182,10 +182,13 @@ class TestFloatingIPs(BaseFloatingIPTest):
self.assertEqual(flip['floating_ip_address'],
get_flip['floating_ip_address'])
def test_delete_floating_ip(self):
@mock.patch('quark.billing.notify')
@mock.patch('quark.billing.build_payload', return_value={})
def test_delete_floating_ip(self, notify, build_payload):
floating_ip = dict(
floating_network_id=self.floating_network.id,
port_id=self.user_port1['id']
port_id=self.user_port1['id'],
address_type='floating'
)
flip = self.plugin.create_floatingip(
self.context, {"floatingip": floating_ip})

4
quark/tests/functional/plugin_modules/test_scaling_ips.py
View File

@@ -208,7 +208,9 @@ class TestScalingIP(test_floating_ips.BaseFloatingIPTest):
msg="Request to the unicorn API is not what is "
"expected.")
def test_delete_scaling_ip(self):
@mock.patch('quark.billing.notify')
@mock.patch('quark.billing.build_payload', return_value={})
def test_delete_scaling_ip(self, notify, build_payload):
scaling_ip = dict(
scaling_network_id=self.scaling_network.id,
ports=[dict(port_id=self.user_port1['id']),

42
quark/tests/plugin_modules/test_floating_ips.py
View File

@@ -41,10 +41,13 @@ class TestRemoveFloatingIPs(test_quark_plugin.TestQuarkPlugin):
mock.patch("quark.db.api.ip_address_deallocate"),
mock.patch("quark.ipam.QuarkIpam.deallocate_ip_address"),
mock.patch("quark.drivers.unicorn_driver.UnicornDriver"
".remove_floating_ip")
".remove_floating_ip"),
mock.patch("quark.billing.notify"),
mock.patch("quark.billing.build_payload")
) as (flip_find, db_fixed_ip_disassoc, db_port_disassoc, db_dealloc,
mock_dealloc, mock_remove_flip):
mock_dealloc, mock_remove_flip, notify, build_payload):
flip_find.return_value = flip_model
build_payload.return_value = {'respek': '4reelz'}
yield
def test_delete_floating_by_ip_address_id(self):
@@ -233,15 +236,18 @@ class TestCreateFloatingIPs(test_quark_plugin.TestQuarkPlugin):
mock.patch("quark.drivers.unicorn_driver.UnicornDriver"
".register_floating_ip"),
mock.patch("quark.db.api.port_associate_ip"),
mock.patch("quark.db.api.floating_ip_associate_fixed_ip")
mock.patch("quark.db.api.floating_ip_associate_fixed_ip"),
mock.patch("quark.billing.notify"),
mock.patch("quark.billing.build_payload")
) as (flip_find, net_find, port_find, alloc_ip, mock_reg_flip,
port_assoc, fixed_ip_assoc):
port_assoc, fixed_ip_assoc, notify, build_payload):
flip_find.return_value = flip_model
net_find.return_value = net_model
port_find.return_value = port_model
alloc_ip.side_effect = _alloc_ip
port_assoc.side_effect = _port_assoc
fixed_ip_assoc.side_effect = _flip_fixed_ip_assoc
build_payload.return_value = {}
yield
def test_create_with_a_port(self):
@@ -537,6 +543,10 @@ class TestUpdateFloatingIPs(test_quark_plugin.TestQuarkPlugin):
addr.ports = []
return addr
def mock_notify(context, notif_type, flip):
"""We don't want to notify from tests"""
pass
with contextlib.nested(
mock.patch("quark.db.api.floating_ip_find"),
mock.patch("quark.db.api.port_find"),
@@ -549,16 +559,20 @@ class TestUpdateFloatingIPs(test_quark_plugin.TestQuarkPlugin):
mock.patch("quark.db.api.port_associate_ip"),
mock.patch("quark.db.api.port_disassociate_ip"),
mock.patch("quark.db.api.floating_ip_associate_fixed_ip"),
mock.patch("quark.db.api.floating_ip_disassociate_fixed_ip")
mock.patch("quark.db.api.floating_ip_disassociate_fixed_ip"),
mock.patch("quark.billing.notify")
) as (flip_find, port_find, reg_flip, update_flip, rem_flip,
port_assoc, port_dessoc, flip_assoc, flip_dessoc):
port_assoc, port_dessoc, flip_assoc, flip_dessoc, notify):
flip_find.return_value = flip_model
port_find.side_effect = _find_port
port_assoc.side_effect = _port_assoc
port_dessoc.side_effect = _port_dessoc
flip_assoc.side_effect = _flip_assoc
flip_dessoc.side_effect = _flip_disassoc
yield
notify.side_effect = mock_notify
# We'll yield a notify to check how many times and with which
# arguments it was called.
yield notify
def test_update_with_new_port_and_no_previous_port(self):
new_port = dict(id="2")
@@ -575,12 +589,14 @@ class TestUpdateFloatingIPs(test_quark_plugin.TestQuarkPlugin):
flip = dict(id="3", fixed_ip_address="172.16.1.1", address=int(addr),
address_readable=str(addr))
with self._stubs(flip=flip, new_port=new_port, ips=ips):
with self._stubs(flip=flip, new_port=new_port, ips=ips) as notify:
content = dict(port_id=new_port["id"])
ret = self.plugin.update_floatingip(self.context, flip["id"],
dict(floatingip=content))
self.assertEqual(ret["fixed_ip_address"], "192.168.0.1")
self.assertEqual(ret["port_id"], new_port["id"])
notify.assert_called_once_with(self.context, 'ip.associate',
mock.ANY)
def test_update_with_new_port(self):
curr_port = dict(id="1")
@@ -598,12 +614,16 @@ class TestUpdateFloatingIPs(test_quark_plugin.TestQuarkPlugin):
address_readable=str(addr))
with self._stubs(flip=flip, curr_port=curr_port,
new_port=new_port, ips=ips):
new_port=new_port, ips=ips) as notify:
content = dict(port_id=new_port["id"])
ret = self.plugin.update_floatingip(self.context, flip["id"],
dict(floatingip=content))
self.assertEqual(ret["fixed_ip_address"], "192.168.0.1")
self.assertEqual(ret["port_id"], new_port["id"])
self.assertEqual(notify.call_count, 2, 'Should notify twice here')
call_list = [mock.call(self.context, 'ip.disassociate', mock.ANY),
mock.call(self.context, 'ip.associate', mock.ANY)]
notify.assert_has_calls(call_list, any_order=True)
def test_update_with_no_port(self):
curr_port = dict(id="1")
@@ -611,12 +631,14 @@ class TestUpdateFloatingIPs(test_quark_plugin.TestQuarkPlugin):
flip = dict(id="3", fixed_ip_address="172.16.1.1", address=int(addr),
address_readable=str(addr))
with self._stubs(flip=flip, curr_port=curr_port):
with self._stubs(flip=flip, curr_port=curr_port) as notify:
content = dict(port_id=None)
ret = self.plugin.update_floatingip(self.context, flip["id"],
dict(floatingip=content))
self.assertEqual(ret.get("fixed_ip_address"), None)
self.assertEqual(ret.get("port_id"), None)
notify.assert_called_once_with(self.context, 'ip.disassociate',
mock.ANY)
def test_update_with_non_existent_port_should_fail(self):
addr = netaddr.IPAddress("10.0.0.1")

162
quark/tests/test_billing.py Normal file
View File

@@ -0,0 +1,162 @@
# Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import json
from oslo_config import cfg
from quark import billing
from quark.db.models import IPAddress
from quark import network_strategy
from quark.tests import test_base
class QuarkBillingBaseTest(test_base.TestBase):
def setUp(self):
super(QuarkBillingBaseTest, self).setUp()
self.strategy = {"00000000-0000-0000-0000-000000000000":
{"bridge": "publicnet",
"subnets": {"4": "public_v4",
"6": "public_v6"}}}
strategy_json = json.dumps(self.strategy)
cfg.CONF.set_override("default_net_strategy", strategy_json, "QUARK")
network_strategy.STRATEGY.load()
# Need to patch it here because billing has loaded before this
# code is executed and had no config available to it.
billing.PUBLIC_NETWORK_ID = \
network_strategy.STRATEGY.get_public_net_id()
TENANT_ID = u'12345'
IP_ID = 'ffffffff-dddd-cccc-bbbb-aaaaaaaaaaaa'
IP_READABLE = '1.1.1.1'
SUBNET_ID = 'badc0ffe-dead-beef-c0fe-baaaaadc0ded'
PUB_NETWORK_ID = '00000000-0000-0000-0000-000000000000'
def get_fake_fixed_address():
ipaddress = IPAddress()
ipaddress.used_by_tenant_id = TENANT_ID
ipaddress.version = 4
ipaddress.id = IP_ID
ipaddress.address_readable = IP_READABLE
ipaddress.subnet_id = SUBNET_ID
ipaddress.network_id = PUB_NETWORK_ID
return ipaddress
class QuarkBillingPayloadTest(QuarkBillingBaseTest):
"""Tests for payload generation.
This is the payload json:
{
'event_type': unicode(EVENT_TYPE_2_CLOUDFEEDS[event_type]),
'tenant_id': unicode(ipaddress.used_by_tenant_id),
'ip_address': unicode(ipaddress.address_readable),
'subnet_id': unicode(ipaddress.subnet_id),
'network_id': unicode(ipaddress.network_id),
'public': True if ipaddress.network_id == PUBLIC_NETWORK_ID else False,
'ip_version': int(ipaddress.version),
'ip_type': unicode(ipaddress.address_type),
'id': unicode(ipaddress.id)
}
"""
def setUp(self):
super(QuarkBillingPayloadTest, self).setUp()
def test_fixed_payload(self):
start_time = datetime.datetime.utcnow().replace(microsecond=0) -\
datetime.timedelta(days=1)
end_time = datetime.datetime.utcnow().replace(microsecond=0)
ipaddress = get_fake_fixed_address()
ipaddress.allocated_at = start_time
ipaddress.deallocated_at = end_time
ipaddress.address_type = 'fixed'
payload = billing.build_payload(ipaddress, 'ip.exists',
start_time=start_time,
end_time=end_time)
self.assertEqual(payload['event_type'], u'USAGE',
'event_type is wrong')
self.assertEqual(payload['tenant_id'], TENANT_ID,
'tenant_id is wrong')
self.assertEqual(payload['ip_address'], IP_READABLE,
'ip_address is wrong')
self.assertEqual(payload['subnet_id'], SUBNET_ID,
'subnet_id is wrong')
self.assertEqual(payload['network_id'], PUB_NETWORK_ID,
'network_id is wrong')
self.assertEqual(payload['public'], True,
'public should be true')
self.assertEqual(payload['ip_version'], 4,
'ip_version should be 4')
self.assertEqual(payload['ip_type'], 'fixed',
'ip_type should be fixed')
self.assertEqual(payload['id'], IP_ID, 'ip_id is wrong')
self.assertEqual(payload['startTime'],
billing.convert_timestamp(start_time),
'startTime is wrong')
self.assertEqual(payload['endTime'],
billing.convert_timestamp(end_time),
'endTime is wrong')
def test_associate_flip_payload(self):
event_time = datetime.datetime.utcnow().replace(microsecond=0)
ipaddress = get_fake_fixed_address()
# allocated_at and deallocated_at could be anything for testing this
ipaddress.allocated_at = event_time
ipaddress.deallocated_at = event_time
ipaddress.address_type = 'floating'
payload = billing.build_payload(ipaddress, 'ip.associate',
event_time=event_time)
self.assertEqual(payload['event_type'], u'UP', 'event_type is wrong')
self.assertEqual(payload['tenant_id'], TENANT_ID, 'tenant_id is wrong')
self.assertEqual(payload['ip_address'], IP_READABLE,
'ip_address is wrong')
self.assertEqual(payload['subnet_id'], SUBNET_ID, 'subnet_id is wrong')
self.assertEqual(payload['network_id'], PUB_NETWORK_ID,
'network_id is wrong')
self.assertEqual(payload['public'], True, 'public should be true')
self.assertEqual(payload['ip_version'], 4, 'ip_version should be 4')
self.assertEqual(payload['ip_type'], 'floating',
'ip_type should be fixed')
self.assertEqual(payload['id'], IP_ID, 'ip_id is wrong')
self.assertEqual(payload['eventTime'],
billing.convert_timestamp(event_time),
'eventTime is wrong')
def test_disassociate_flip_payload(self):
event_time = datetime.datetime.utcnow().replace(microsecond=0)
ipaddress = get_fake_fixed_address()
# allocated_at and deallocated_at could be anything for testing this
ipaddress.allocated_at = event_time
ipaddress.deallocated_at = event_time
ipaddress.address_type = 'floating'
payload = billing.build_payload(ipaddress, 'ip.disassociate',
event_time=event_time)
self.assertEqual(payload['event_type'], u'DOWN', 'event_type is wrong')
self.assertEqual(payload['tenant_id'], TENANT_ID, 'tenant_id is wrong')
self.assertEqual(payload['ip_address'], IP_READABLE,
'ip_address is wrong')
self.assertEqual(payload['subnet_id'], SUBNET_ID, 'subnet_id is wrong')
self.assertEqual(payload['network_id'], PUB_NETWORK_ID,
'network_id is wrong')
self.assertEqual(payload['public'], True, 'public should be true')
self.assertEqual(payload['ip_version'], 4, 'ip_version should be 4')
self.assertEqual(payload['ip_type'], 'floating',
'ip_type should be fixed')
self.assertEqual(payload['id'], IP_ID, 'ip_id is wrong')
self.assertEqual(payload['eventTime'],
billing.convert_timestamp(event_time),
'eventTime is wrong')

95
quark/tests/test_ipam.py
View File

@@ -24,6 +24,7 @@ from neutron.common import rpc
from neutron_lib import exceptions as n_exc
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_utils import timeutils
from quark.db import models
from quark import exceptions as q_exc
@@ -359,7 +360,8 @@ class QuarkIPAddressDeallocation(QuarkIpamBaseTest):
def test_deallocate_ips_by_port(self):
port_dict = dict(ip_addresses=[], device_id="foo")
addr_dict = dict(subnet_id=1, address_readable=None,
created_at=None, used_by_tenant_id=1)
created_at=None, used_by_tenant_id=1,
version=4)
port = models.Port()
port.update(port_dict)
@@ -379,7 +381,7 @@ class QuarkIPAddressDeallocation(QuarkIpamBaseTest):
port_dict = dict(ip_addresses=[], device_id="foo")
addr_dict = dict(subnet_id=1, address_readable="0.0.0.0",
created_at=None, used_by_tenant_id=1,
address=0)
address=0, version=4)
port = models.Port()
port.update(port_dict)
@@ -650,6 +652,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest):
address["subnet"] = models.Subnet(cidr="0.0.0.0/24", first_ip=first,
last_ip=last,
next_auto_assign_ip=first)
address["allocated_at"] = timeutils.utcnow()
with self._stubs(subnets=[[(subnet6, 0)]],
addresses=[address, None, None]) as addr_realloc:
address = []
@@ -682,6 +685,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest):
address["address"] = self.v46_val
address["version"] = 4
address["subnet"] = models.Subnet(cidr="0.0.0.0/24")
address["allocated_at"] = timeutils.utcnow()
with self._stubs(subnets=[[(subnet6, 0)]],
addresses=[address, None, None]) as addr_realloc:
address = []
@@ -709,6 +713,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest):
address["address"] = self.v46_val
address["version"] = 4
address["subnet"] = models.Subnet(cidr="0.0.0.0/24")
address["allocated_at"] = timeutils.utcnow()
with self._stubs(subnets=[[(subnet6, 0)]],
addresses=[address, None, None]) as addr_realloc:
address = []
@@ -737,6 +742,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest):
address["address"] = self.v46_val
address["version"] = 4
address["subnet"] = models.Subnet(cidr="0.0.0.0/24")
address["allocated_at"] = timeutils.utcnow()
with self._stubs(subnets=[[]],
addresses=[address, None, None]) as addr_realloc:
address = []
@@ -758,6 +764,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest):
address["address"] = self.v46_val
address["version"] = 4
address["subnet"] = models.Subnet(cidr="::ffff:0:0/96")
address["allocated_at"] = timeutils.utcnow()
mac = models.MacAddress()
mac["address"] = netaddr.EUI("AA:BB:CC:DD:EE:FF")
@@ -792,6 +799,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest):
address["address"] = self.v46_val
address["version"] = 4
address["subnet"] = models.Subnet(cidr="::ffff:0:0/96")
address["allocated_at"] = timeutils.utcnow()
mac = models.MacAddress()
mac["address"] = netaddr.EUI("AA:BB:CC:DD:EE:FF")
@@ -816,6 +824,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest):
address["address"] = str(self.v46_val)
address["version"] = 6
address["subnet"] = models.Subnet(cidr="::ffff:0:0/96")
address["allocated_at"] = timeutils.utcnow()
with self._stubs(subnets=[[(subnet4, 0)]],
addresses=[address, None, None]) as addr_realloc:
addresses = []
@@ -842,6 +851,7 @@ class QuarkIpamTestBothIpAllocation(QuarkIpamBaseTest):
address1["address"] = self.v46_val
address1["version"] = 4
address1["subnet"] = models.Subnet(cidr="0.0.0.0/24")
address1["allocated_at"] = timeutils.utcnow()
with self._stubs(subnets=[[(subnet6, 1)]],
addresses=[address1]) as addr_realloc:
@@ -982,6 +992,7 @@ class QuarkIpamTestBothRequiredIpAllocation(QuarkIpamBaseTest):
address["address"] = 4
address["version"] = 4
address["subnet"] = models.Subnet(cidr="0.0.0.0/24")
address["allocated_at"] = timeutils.utcnow()
with self._stubs(subnets=[[(subnet6, 0)]],
addresses=[address, None, None]) as addr_realloc:
address = []
@@ -1007,6 +1018,7 @@ class QuarkIpamTestBothRequiredIpAllocation(QuarkIpamBaseTest):
address1["address"] = self.v46_val
address1["version"] = 4
address1["subnet"] = models.Subnet(cidr="0.0.0.0/24")
address1["allocated_at"] = timeutils.utcnow()
with self._stubs(subnets=[[(subnet6, 0)]],
addresses=[address1]) as addr_realloc:
@@ -1111,6 +1123,8 @@ class QuarkIpamAllocateFromV6Subnet(QuarkIpamBaseTest):
ip_mod = models.IPAddress()
ip_mod["address"] = ip_address.value
ip_mod["deallocated"] = deallocated
ip_mod["allocated_at"] = timeutils.utcnow()
ip_mod["version"] = ip_address.version
with contextlib.nested(
mock.patch("quark.db.models.IPPolicy.get_cidrs_ip_set"),
@@ -1156,7 +1170,7 @@ class QuarkIpamAllocateFromV6Subnet(QuarkIpamBaseTest):
def test_allocate_v6_with_ip_and_no_mac(self):
fip = netaddr.IPAddress('fe80::')
ip_address = netaddr.IPAddress("fe80::7")
lip = netaddr.IPAddress('fe80::FF:FFFF')
lip = netaddr.IPAddress('feed::FF:FFFF')
port_id = "945af340-ed34-4fec-8c87-853a2df492b4"
subnet6 = dict(id=1, first_ip=fip, last_ip=lip,
cidr="feed::/104", ip_version=6,
@@ -1482,12 +1496,11 @@ class QuarkIPAddressAllocationTestRetries(QuarkIpamBaseTest):
with contextlib.nested(
mock.patch("quark.db.api.ip_address_find"),
mock.patch("quark.db.api.ip_address_create"),
mock.patch("quark.ipam.QuarkIpam._notify_new_addresses"),
mock.patch("quark.db.api.subnet_find_ordered_by_most_full"),
mock.patch("quark.db.api.subnet_update_next_auto_assign_ip"),
mock.patch("quark.db.api.subnet_update_set_full"),
mock.patch("sqlalchemy.orm.session.Session.refresh")
) as (addr_find, addr_create, notify, subnet_find, subnet_update,
) as (addr_find, addr_create, subnet_find, subnet_update,
subnet_set_full, refresh):
addr_find.side_effect = [None, None, None]
addr_mods = []
@@ -1520,7 +1533,10 @@ class QuarkIPAddressAllocationTestRetries(QuarkIpamBaseTest):
cidr="0.0.0.0/24", ip_version=4,
ip_policy=None)
subnets = [(subnet1, 1)]
addr_found = dict(id=1, address=2)
addr_found = dict(id=1,
address=2,
allocated_at=timeutils.utcnow(),
version=4)
with self._stubs(subnets=subnets,
address=[q_exc.IPAddressRetryableFailure,
addr_found]) as (sub_mods, addr_mods):
@@ -1550,7 +1566,8 @@ class QuarkIPAddressAllocationTestRetries(QuarkIpamBaseTest):
cidr="::/64", ip_version=6,
ip_policy=None)
subnets = [(subnet1, 1), (subnet1, 1)]
addr_found = dict(id=1, address=1)
addr_found = dict(id=1, address=1, version=4,
allocated_at=timeutils.utcnow())
with self._stubs(
subnets=subnets,
@@ -1569,7 +1586,8 @@ class QuarkIPAddressAllocationTestRetries(QuarkIpamBaseTest):
cidr="0.0.0.0/24", ip_version=4,
ip_policy=None)
subnets = [(subnet1, 1), (subnet1, 1)]
addr_found = dict(id=1, address=256)
addr_found = dict(id=1, address=256, version=4,
allocated_at=timeutils.utcnow())
with self._stubs(subnets=subnets,
address=[q_exc.IPAddressRetryableFailure,
addr_found]) as (sub_mods, addr_mods):
@@ -1584,7 +1602,8 @@ class QuarkIPAddressAllocationTestRetries(QuarkIpamBaseTest):
ip_policy=None,
do_not_use=1)
subnets = []
addr_found = dict(id=1, address=256)
addr_found = dict(id=1, address=256, version=4,
allocated_at=timeutils.utcnow())
with self._stubs(subnets=subnets,
address=[q_exc.IPAddressRetryableFailure,
addr_found]) as (sub_mods, addr_mods):
@@ -1600,7 +1619,8 @@ class QuarkIPAddressAllocationTestRetries(QuarkIpamBaseTest):
cidr="0.0.0.0/31", ip_version=4,
ip_policy=None)
subnets = [(subnet1, 1)]
addr_found = dict(id=1, address=1)
addr_found = dict(id=1, address=1, version=4,
allocated_at=timeutils.utcnow())
with self._stubs(subnets=subnets, address=[addr_found]) as (sub_mods,
addr_mods):
addr = []
@@ -1810,12 +1830,35 @@ class QuarkIPAddressAllocationNotifications(QuarkIpamBaseTest):
yield notify
def test_allocation_notification(self):
"""Tests IP allocation
Notification payload looks like this:
{
'ip_type': u'fixed',
'id': u'ee267779-e513-4132-a9ba-55148eab584f',
'event_type': u'CREATE',
'eventTime': u'2016-05-26T21:47:45.722735Z',
'network_id': u'None',
'tenant_id': u'1',
'subnet_id': u'1',
'public': False,
'ip_address': u'0.0.0.0',
'ip_version': 4
}
But for simplicity replaced it with mock.ANY
"""
subnet = dict(id=1, first_ip=0, last_ip=255,
cidr="0.0.0.0/24", ip_version=4,
next_auto_assign_ip=1,
ip_policy=None)
allocated_at = timeutils.utcnow()
deallocated_at = timeutils.utcnow()
address = dict(address=0, created_at="123", subnet_id=1,
address_readable="0.0.0.0", used_by_tenant_id=1)
address_readable="0.0.0.0", used_by_tenant_id=1,
version=4, allocated_at=allocated_at,
deallocated_at=deallocated_at,
address_type='fixed')
with self._stubs(
address,
subnets=[(subnet, 1)],
@@ -1827,16 +1870,13 @@ class QuarkIPAddressAllocationNotifications(QuarkIpamBaseTest):
notify.assert_called_once_with("network")
notify.return_value.info.assert_called_once_with(
self.context,
"ip_block.address.create",
dict(ip_block_id=address["subnet_id"],
ip_address="0.0.0.0",
device_ids=[],
created_at=address["created_at"],
used_by_tenant_id=1))
"ip.add",
mock.ANY)
def test_deallocation_notification(self):
addr_dict = dict(address=0, created_at="123", subnet_id=1,
address_readable="0.0.0.0", used_by_tenant_id=1)
address_readable="0.0.0.0", used_by_tenant_id=1,
version=4)
address = models.IPAddress()
address.update(addr_dict)
@@ -1847,16 +1887,15 @@ class QuarkIPAddressAllocationNotifications(QuarkIpamBaseTest):
with self._stubs(dict(), deleted_at="456") as notify:
self.ipam.deallocate_ips_by_port(self.context, port)
notify.assert_called_once_with("network")
notify.return_value.info.assert_called_once_with(
self.context,
"ip_block.address.delete",
dict(ip_block_id=address["subnet_id"],
ip_address="0.0.0.0",
device_ids=[],
created_at=address["created_at"],
deleted_at="456",
used_by_tenant_id=1))
notify.assert_called_with('network')
self.assertEqual(notify.call_count, 2,
'Should have called notify twice')
# When we deallocate an IP we must send a usage message as well
# Verify that we called both methods. Order matters.
call_list = [mock.call(self.context, 'ip.delete', mock.ANY),
mock.call(self.context, 'ip.exists', mock.ANY)]
notify.return_value.info.assert_has_calls(call_list,
any_order=False)
class QuarkIpamTestV6IpGeneration(QuarkIpamBaseTest):

122
quark/tools/billing.py Normal file
View File

@@ -0,0 +1,122 @@
# Copyright 2016 Openstack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Calculations for different cases for additional IP billing
See notes in quark/billing.py for more details.
"""
import click
import datetime
from neutron.common import config
from neutron import context as neutron_context
from pprint import pprint as pp
from quark import billing
from quark.db import models
from quark import network_strategy
def make_case2(context):
"""This is a helper method for testing.
When run with the current context, it will create a case 2 entries
in the database. See top of file for what case 2 is.
"""
query = context.session.query(models.IPAddress)
period_start, period_end = billing.calc_periods()
ip_list = billing.build_full_day_ips(query, period_start, period_end)
import random
ind = random.randint(0, len(ip_list) - 1)
address = ip_list[ind]
address.allocated_at = datetime.datetime.utcnow() -\
datetime.timedelta(days=1)
context.session.add(address)
context.session.flush()
@click.command()
@click.option('--notify', is_flag=True,
help='If true, sends notifications to billing')
@click.option('--hour', default=0,
help='period start hour, e.g. 0 is midnight')
@click.option('--minute', default=0,
help='period start minute, e.g. 0 is top of the hour')
def main(notify, hour, minute):
"""Runs billing report. Optionally sends notifications to billing"""
# Read the config file and get the admin context
config_opts = ['--config-file', '/etc/neutron/neutron.conf']
config.init(config_opts)
# Have to load the billing module _after_ config is parsed so
# that we get the right network strategy
network_strategy.STRATEGY.load()
billing.PUBLIC_NETWORK_ID = network_strategy.STRATEGY.get_public_net_id()
config.setup_logging()
context = neutron_context.get_admin_context()
# A query to get all IPAddress objects from the db
query = context.session.query(models.IPAddress)
(period_start, period_end) = billing.calc_periods(hour, minute)
full_day_ips = billing.build_full_day_ips(query,
period_start,
period_end)
partial_day_ips = billing.build_partial_day_ips(query,
period_start,
period_end)
if notify:
# '==================== Full Day ============================='
for ipaddress in full_day_ips:
click.echo('start: {}, end: {}'.format(period_start, period_end))
payload = billing.build_payload(ipaddress,
'ip.exists',
start_time=period_start,
end_time=period_end)
billing.do_notify(context,
'ip.exists',
payload)
# '==================== Part Day ============================='
for ipaddress in partial_day_ips:
click.echo('start: {}, end: {}'.format(period_start, period_end))
payload = billing.build_payload(ipaddress,
'ip.exists',
start_time=ipaddress.allocated_at,
end_time=period_end)
billing.do_notify(context,
'ip.exists',
payload)
else:
click.echo('Case 1 ({}):\n'.format(len(full_day_ips)))
for ipaddress in full_day_ips:
pp(billing.build_payload(ipaddress,
'ip.exists',
start_time=period_start,
end_time=period_end))
click.echo('\n===============================================\n')
click.echo('Case 2 ({}):\n'.format(len(partial_day_ips)))
for ipaddress in partial_day_ips:
pp(billing.build_payload(ipaddress,
'ip.exists',
start_time=ipaddress.allocated_at,
end_time=period_end))
if __name__ == '__main__':
main()

1
requirements.txt
View File

@@ -1,5 +1,6 @@
SQLAlchemy<1.1.0,>=1.0.10
alembic==0.8.2
click>=6.6
neutron-lib>=0.0.1
oslo.concurrency
oslo.config