openstack/cloudkitty
Code Issues Proposed changes
5e716e1429
cloudkitty/contrib/ci/csv_writer.py
Luka Peschke 9c88e50b73 Add a custom JSONEncoder 
A custom JSONEncoder allowing to serialize decimal.Decimal instances
is defined in several places in the codebase. This provides a common
encoder class that supports decimal.Decimal and datetime.datetime objects.

Change-Id: Ie65b1f97f2eeb7a8bc6203576bdbc50b109867cc
2019-02-13 15:05:17 +01:00

613 lines
17 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2015 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
import calendar
import copy
import csv
import datetime
import random
import sys
import uuid
from cloudkitty import json_utils as json
COMPUTE = {
"type": "compute",
"desc": {},
"vol": {
"qty": 1,
"unit": "instance"}}
COMPUTE_RESOURCE = {
"availability_zone": "nova",
"flavor": "m1.nano",
"image_id": "f5600101-8fa2-4864-899e-ebcb7ed6b568",
"memory": "64",
"metadata": {
"farm": "prod"},
"name": "prod1",
"project_id": "f266f30b11f246b589fd266f85eeec39",
"user_id": "55b3379b949243009ee96972fbf51ed1",
"vcpus": "1"}
IMAGE = {
"type": "image",
"desc": {},
"vol": {
"qty": 214106112.0,
"unit": "B"}}
IMAGE_RESOURCE = {
"name": "cirros-0.3.4-x86_64-uec-ramdisk",
"checksum": "be575a2b939972276ef675752936977f",
"disk_format": "ari",
"protected": "False",
"container_format": "ari",
"min_disk": "0",
"is_public": "True",
"min_ram": "0",
"project_id": "f1873b13951542268bf7eed7cf971e52",
"resource_id": "08017fbc-b13a-4d8d-b002-4eb4eff54cd4",
"source": "openstack",
"user_id": "None",
"size": "214106112"}
VOLUME = {
"type": "volume",
"desc": {},
"vol": {
"qty": 1,
"unit": "GB"}}
VOLUME_RESOURCE = {
'instance_uuid': 'None',
'status': 'available',
'display_name': 'test-vol',
'event_type': 'volume.create.end',
'availability_zone': 'nova',
'tenant_id': 'cd27b013b9db4f4099e273e4b9949023',
'created_at': '2015-04-28 13:34:25',
'snapshot_id': 'None',
'volume_type': '314150bc-221f-4676-a7cf-16f12850b217',
'host': 'volume.devstack@lvmdriver-1#lvmdriver-1',
'replication_driver_data': 'None',
'replication_status': 'disabled',
'volume_id': '2bed6a3d-468a-459b-802b-44930016c0a3',
'replication_extended_status': 'None',
'user_id': '2524d5a52ce64a569d131d7dc1dfb455',
'launched_at': '2015-04-28 13:34:26.869928',
'size': '1',
"project_id": "f1873b13951542268bf7eed7cf971e52",
"resource_id": "08017fbc-b13a-4d8d-b002-4eb4eff54cd4",
"source": "openstack",
"user_id": "None"}
NETWORK_BW_IN = {
"type": "network.bw.in",
"desc": {},
"vol": {
"qty": 4546.0,
"unit": "B"}}
NETWORK_BW_OUT = {
"type": "network.bw.out",
"desc": {},
"vol": {
"qty": 50.0,
"unit": "MB"}}
NETWORK_BW_RESOURCE = {
'instance_id': 'eef9673d-5d24-43fd-89f5-2929acc7e193',
'instance_type': '42',
'mac': 'fa:16:3e:dd:b2:80',
'fref': 'None',
'name': 'tap12a7d4e1-fc',
"project_id": "f1873b13951542268bf7eed7cf971e52",
"resource_id": "08017fbc-b13a-4d8d-b002-4eb4eff54cd4",
"source": "openstack",
"user_id": "None"}
FLOATING = {
"type": "network.floating",
"desc": {},
"vol": {
"qty": 1.0,
"unit": "ip"}}
FLOATING_RESOURCE = {
'router_id': '3d9b2725-fc90-43d0-b119-630e80e1ec51',
'status': 'DOWN',
'event_type': 'floatingip.update.end',
'tenant_id': 'cd27b013b9db4f4099e273e4b9949023',
'floating_network_id': '14198fb4-dc96-45fb-9dde-546f6b0f892f',
'host': 'network.devstack',
'fixed_ip_address': '10.0.0.5',
'floating_ip_address': '172.24.4.3',
'port_id': '12a7d4e1-fcc3-4a3c-8e57-c4baf7787b57',
"project_id": "cd27b013b9db4f4099e273e4b9949023",
"resource_id": "ebf6485d-7f6f-4c67-97f7-7896324e12d4",
"source": "openstack",
"user_id": "7319b5d1269d4166a402868b570aad19",
'id': 'ebf6485d-7f6f-4c67-97f7-7896324e12d4'}
class VariationMapper(object):
day_map = {
0: 'mon',
1: 'tue',
2: 'wed',
3: 'thu',
4: 'fri',
5: 'sat',
6: 'sun'}
var_map = {
'mon': {},
'tue': {},
'wed': {},
'thu': {},
'fri': {},
'sat': {},
'sun': {}}
def __init__(self, default=1.0):
self.default = default
def get_var(self, dt):
weekday = self.day_map[dt.weekday()]
if weekday in self.var_map:
wday_map = self.var_map[weekday]
if dt.hour in wday_map:
return wday_map[dt.hour]
elif 'default' in wday_map:
return wday_map['default']
elif 'default' in self.var_map:
return self.var_map['default']
return self.default
class VolumeVariationMapper(VariationMapper):
def get_vol(self, dt):
value = self.get_var(dt)
var_value = value * 0.1
return random.gauss(value, var_value)
class BaseGenerator(object):
base_sample = None
base_resource = None
rand = True
field_maps = {}
def __init__(self, nb_res=1, var_map=None, vol_map=None):
self.nb_res = nb_res
self.var_map = var_map if var_map else VariationMapper()
self.vol_map = vol_map
self.init_mapper()
self.resources = []
def init_mapper(self):
pass
def generate_resources(self):
for i in range(self.nb_res):
res = copy.deepcopy(self.base_resource)
for field, mapping in self.field_maps.items():
if hasattr(self, mapping):
mapping = getattr(self, mapping)
if isinstance(mapping, dict):
if self.rand:
value = random.choice(mapping.keys())
else:
value = mapping.keys()[i]
res[field] = value
for k, v in mapping[value].items():
res[k] = v
elif isinstance(mapping, list):
if self.rand:
value = random.choice(mapping)
else:
value = mapping[i]
res[field] = value
elif callable(mapping):
res[field] = mapping(i)
else:
res[field] = mapping
self.resources.append(res)
def generate_samples(self, dt):
samples = []
res_var = int(self.var_map.get_var(dt))
for i in range(res_var):
sample = copy.deepcopy(self.base_sample)
sample['desc'] = self.resources[i]
if self.vol_map:
qty = self.vol_map.get_vol(dt)
sample['vol']['qty'] = qty
elif 'size' in sample['desc']:
sample['vol']['qty'] = sample['desc']['size']
# Packing
sample['desc'] = json.dumps(self.resources[i])
sample['vol'] = json.dumps(sample['vol'])
samples.append(sample)
return samples
class ComputeVarMapper(VariationMapper):
var_map = {
'mon': {
'default': 1.0,
12: 2.0,
13: 3.0,
14: 2.0,
18: 2.0,
19: 3.0,
20: 4.0,
21: 4.0,
22: 3.0,
23: 2.0,
},
'tue': {
'default': 1.0,
12: 2.0,
13: 3.0,
14: 2.0,
18: 2.0,
19: 3.0,
20: 4.0,
21: 4.0,
22: 3.0,
23: 2.0,
},
'wed': {
'default': 1.0,
12: 2.0,
13: 3.0,
14: 2.0,
18: 2.0,
19: 3.0,
20: 4.0,
21: 4.0,
22: 3.0,
23: 2.0,
},
'thu': {
'default': 1.0,
12: 2.0,
13: 3.0,
14: 2.0,
18: 2.0,
19: 3.0,
20: 4.0,
21: 4.0,
22: 3.0,
23: 2.0,
},
'fri': {
'default': 1.0,
12: 2.0,
13: 3.0,
14: 2.0,
18: 2.0,
19: 3.0,
20: 4.0,
21: 4.0,
22: 3.0,
23: 2.0,
},
'sat': {
'default': 2.0,
12: 3.0,
13: 4.0,
14: 3.0,
18: 3.0,
19: 4.0,
20: 4.0,
21: 4.0,
22: 4.0,
23: 3.0,
},
'sun': {
'default': 2.0,
12: 3.0,
13: 4.0,
14: 3.0,
18: 3.0,
19: 4.0,
20: 4.0,
21: 4.0,
22: 4.0,
23: 3.0,
}}
class ComputeGenerator(BaseGenerator):
base_sample = COMPUTE
base_resource = COMPUTE_RESOURCE
field_maps = {'flavor': 'flavors',
'image': 'images',
'name': 'generate_name',
'resource_id': 'res_id'}
def init_mapper(self):
self.flavors = {
'm1.nano': {
'vcpus': '1',
'memory': '64'},
'm1.micro': {
'vcpus': '1',
'memory': '128'}}
self.images = []
self.res_id = []
for i in range(self.nb_res):
self.res_id.append(str(uuid.uuid1()))
def generate_name(self, *args):
basename = 'instance{}'
return basename.format(args[0])
class ImageGenerator(BaseGenerator):
base_sample = IMAGE
base_resource = IMAGE_RESOURCE
field_maps = {'name': 'images'}
rand = False
def init_mapper(self):
self.images = {
'cirros-0.3.4-x86_64-uec-kernel': {
'checksum': '836c69cbcd1dc4f225daedbab6edc7c7',
'disk_format': 'ari',
'container_format': 'ari',
'size': '4969360',
'resource_id': '5dd34048-6eeb-4b6c-aa51-62487733e5a1'},
'cirros-0.3.4-x86_64-uec-ramdisk': {
'checksum': '68085af2609d03e51c7662395b5b6e4b',
'disk_format': 'aki',
'container_format': 'aki',
'size': '3723817',
'resource_id': 'e512a97d-1ed5-4b27-a55a-1b9e5087936a'},
'Fedora-x86_64-20-20131211.1-sda': {
'checksum': '51bc16b900bf0f814bb6c0c3dd8f0790',
'disk_format': 'qcow2',
'container_format': 'bare',
'size': '214106112',
'resource_id': '3ee99f3f-7ecf-47b2-9a40-6df2d66ef5ae'}}
class VolumeGenerator(BaseGenerator):
base_sample = VOLUME
base_resource = VOLUME_RESOURCE
field_maps = {'volume_id': 'volumes'}
rand = False
def init_mapper(self):
self.volumes = {
'2bed6a3d-468a-459b-802b-44930016c0a3': {
'size': '10'},
'4fd33321-6a5f-4351-94ca-db398cd708e9': {
'size': '20'}}
def generate_name(self, *args):
basename = 'volume{}'
return basename.format(args[0])
class NetBWVolMapper(VolumeVariationMapper):
var_map = {
'mon': {
'default': 1024,
12: 2048,
13: 3072,
14: 2048,
18: 2048,
19: 3072,
20: 4096,
21: 4096,
22: 3072,
23: 2048,
},
'tue': {
'default': 1024,
12: 2048,
13: 3072,
14: 2048,
18: 2048,
19: 3072,
20: 4096,
21: 4096,
22: 3072,
23: 2048,
},
'wed': {
'default': 1024,
12: 2048,
13: 3072,
14: 2048,
18: 2048,
19: 3072,
20: 4096,
21: 4096,
22: 3072,
23: 2048,
},
'thu': {
'default': 1024,
12: 2048,
13: 3072,
14: 2048,
18: 2048,
19: 3072,
20: 4096,
21: 4096,
22: 3072,
23: 2048,
},
'fri': {
'default': 1024,
12: 2048,
13: 3072,
14: 2048,
18: 2048,
19: 3072,
20: 4096,
21: 4096,
22: 3072,
23: 2048,
},
'sat': {
'default': 2048,
12: 3072,
13: 4096,
14: 3072,
18: 3072,
19: 4096,
20: 4096,
21: 4096,
22: 4096,
23: 3072,
},
'sun': {
'default': 2048,
12: 3072,
13: 4096,
14: 3072,
18: 3072,
19: 4096,
20: 4096,
21: 4096,
22: 4096,
23: 3072,
}}
class NetworkBWGenerator(BaseGenerator):
base_sample = NETWORK_BW_IN
base_resource = NETWORK_BW_RESOURCE
field_maps = {'instance_id': 'instances',
'name': 'generate_name',
'mac': 'generate_mac',
'resource_id': 'res_id'}
rand = False
def init_mapper(self):
self.instances = []
self.res_id = []
for i in range(self.nb_res):
self.res_id.append(str(uuid.uuid1()))
def generate_name(self, *args):
basename = 'tap{}-fc'
return basename.format(args[0])
def generate_mac(self, *args):
basemac = 'fa:16:3e:{:0=2x}:{:0=2x}:{:0=2x}'
return basemac.format(
random.randint(1, 255),
random.randint(1, 255),
random.randint(1, 255))
def generate_samples(self, dt):
samples = []
self.base_sample = NETWORK_BW_OUT
samples.extend(super(NetworkBWGenerator, self).generate_samples(dt))
self.base_sample = NETWORK_BW_IN
samples.extend(super(NetworkBWGenerator, self).generate_samples(dt))
return samples
class FloatingGenerator(BaseGenerator):
base_sample = FLOATING
base_resource = FLOATING_RESOURCE
field_maps = {'fixed_ip_address': 'generate_ip_addr',
'floating_ip_address': 'generate_floating_addr',
'port_id': 'generate_port_id',
'resource_id': 'res_id',
'id': 'res_id'}
rand = False
def init_mapper(self):
self.res_id = []
for i in range(self.nb_res):
self.res_id.append(str(uuid.uuid1()))
def generate_name(self, *args):
basename = 'volume{}'
return basename.format(args[0])
def generate_port_id(self, *args):
return str(uuid.uuid1())
def generate_ip_addr(self, *args):
baseip = '10.0.0.{}'
return baseip.format(random.randint(5, 250))
def generate_floating_addr(self, *args):
baseip = '172.24.4.{}'
return baseip.format(random.randint(5, 250))
def write_samples(writer, dt, samples):
for sample in samples:
ts = calendar.timegm(dt.timetuple())
sample['begin'] = ts
sample['end'] = ts + 3600
writer.writerow(sample)
def main():
# Generators
compute_var = ComputeVarMapper()
image = ImageGenerator(3)
image.generate_resources()
volume = VolumeGenerator(2)
volume.generate_resources()
floating = FloatingGenerator(4, compute_var)
floating.generate_resources()
compute = ComputeGenerator(4, compute_var)
compute.images = [resource['resource_id']
for resource in image.resources]
compute.generate_resources()
net_bw = NetworkBWGenerator(4, compute_var, NetBWVolMapper())
net_bw.instances = [resource['resource_id']
for resource in compute.resources]
net_bw.generate_resources()
generators = [compute, image, volume, net_bw, floating]
# Date
now = datetime.datetime.utcnow()
hour_delta = datetime.timedelta(hours=1)
cur_date = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
cur_month = cur_date.month
filename = sys.argv[1] if len(sys.argv) > 1 else 'generated.csv'
with open(filename, 'wb') as csvfile:
writer = csv.DictWriter(csvfile,
['begin', 'end', 'type', 'desc', 'vol'])
writer.writeheader()
while cur_date.month == cur_month:
for generator in generators:
samples = generator.generate_samples(cur_date)
write_samples(writer, cur_date, samples)
cur_date += hour_delta
if __name__ == '__main__':
main()
View Git Blame Copy Permalink