Added several Swift aggregations (including a new usage
component for calculating rate changes). Also fixed some pep8 issues. Change-Id: I46685d39ace663595aa524f04d8d35a71c9432c3
This commit is contained in:
parent
2e04a6e5e0
commit
d8e73f3bde
|
@ -12,10 +12,10 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
from monasca_transform.driver.mon_metrics_kafka import invoke
|
||||||
|
|
||||||
|
|
||||||
activate_this_file = "/opt/monasca/transform/venv/bin/activate_this.py"
|
activate_this_file = "/opt/monasca/transform/venv/bin/activate_this.py"
|
||||||
execfile(activate_this_file, dict(__file__=activate_this_file))
|
execfile(activate_this_file, dict(__file__=activate_this_file))
|
||||||
|
|
||||||
from monasca_transform.driver.mon_metrics_kafka import invoke
|
|
||||||
|
|
||||||
invoke()
|
invoke()
|
||||||
|
|
|
@ -54,7 +54,7 @@ spark_home = /opt/spark/current
|
||||||
spark_python_files = /opt/monasca/transform/lib/monasca-transform.zip
|
spark_python_files = /opt/monasca/transform/lib/monasca-transform.zip
|
||||||
|
|
||||||
# How often the stream should be read (in seconds)
|
# How often the stream should be read (in seconds)
|
||||||
stream_interval = 120
|
stream_interval = 300
|
||||||
|
|
||||||
# The working directory for monasca-transform
|
# The working directory for monasca-transform
|
||||||
work_dir = /var/run/monasca/transform
|
work_dir = /var/run/monasca/transform
|
||||||
|
|
|
@ -14,11 +14,12 @@
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
from monasca_transform.service.transform_service import main_service
|
||||||
|
|
||||||
|
|
||||||
activate_this_file = "/opt/monasca/transform/venv/bin/activate_this.py"
|
activate_this_file = "/opt/monasca/transform/venv/bin/activate_this.py"
|
||||||
execfile(activate_this_file, dict(__file__=activate_this_file))
|
execfile(activate_this_file, dict(__file__=activate_this_file))
|
||||||
|
|
||||||
from monasca_transform.service.transform_service import main_service
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
main_service()
|
main_service()
|
||||||
|
|
|
@ -60,7 +60,7 @@ spark_home = /opt/spark/current
|
||||||
spark_python_files = /opt/stack/monasca-transform/dist/monasca_transform-0.0.1.egg
|
spark_python_files = /opt/stack/monasca-transform/dist/monasca_transform-0.0.1.egg
|
||||||
|
|
||||||
# How often the stream should be read (in seconds)
|
# How often the stream should be read (in seconds)
|
||||||
stream_interval = 120
|
stream_interval = 300
|
||||||
|
|
||||||
# The working directory for monasca-transform
|
# The working directory for monasca-transform
|
||||||
work_dir = /opt/stack/monasca-transform
|
work_dir = /opt/stack/monasca-transform
|
||||||
|
|
|
@ -163,6 +163,25 @@ class RollupQuantity(SetterComponent):
|
||||||
|
|
||||||
transform_spec_df = transform_context.transform_spec_df_info
|
transform_spec_df = transform_context.transform_spec_df_info
|
||||||
|
|
||||||
|
# get rollup operation (sum, max, avg, min)
|
||||||
|
agg_params = transform_spec_df.select(
|
||||||
|
"aggregation_params_map.setter_rollup_operation").\
|
||||||
|
collect()[0].asDict()
|
||||||
|
setter_rollup_operation = agg_params["setter_rollup_operation"]
|
||||||
|
|
||||||
|
instance_usage_trans_df = RollupQuantity.setter_by_operation(
|
||||||
|
transform_context,
|
||||||
|
instance_usage_df,
|
||||||
|
setter_rollup_operation)
|
||||||
|
|
||||||
|
return instance_usage_trans_df
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def setter_by_operation(transform_context, instance_usage_df,
|
||||||
|
setter_rollup_operation):
|
||||||
|
|
||||||
|
transform_spec_df = transform_context.transform_spec_df_info
|
||||||
|
|
||||||
# get fields we want to group by for a rollup
|
# get fields we want to group by for a rollup
|
||||||
agg_params = transform_spec_df.select(
|
agg_params = transform_spec_df.select(
|
||||||
"aggregation_params_map.setter_rollup_group_by_list"). \
|
"aggregation_params_map.setter_rollup_group_by_list"). \
|
||||||
|
@ -178,14 +197,8 @@ class RollupQuantity(SetterComponent):
|
||||||
aggregation_period)
|
aggregation_period)
|
||||||
|
|
||||||
# group by columns list
|
# group by columns list
|
||||||
group_by_columns_list = group_by_period_list + \
|
group_by_columns_list = \
|
||||||
setter_rollup_group_by_list
|
group_by_period_list + setter_rollup_group_by_list
|
||||||
|
|
||||||
# get rollup operation (sum, max, avg, min)
|
|
||||||
agg_params = transform_spec_df.select(
|
|
||||||
"aggregation_params_map.setter_rollup_operation").\
|
|
||||||
collect()[0].asDict()
|
|
||||||
setter_rollup_operation = agg_params["setter_rollup_operation"]
|
|
||||||
|
|
||||||
# perform rollup operation
|
# perform rollup operation
|
||||||
instance_usage_json_rdd = RollupQuantity._rollup_quantity(
|
instance_usage_json_rdd = RollupQuantity._rollup_quantity(
|
||||||
|
|
|
@ -0,0 +1,148 @@
|
||||||
|
# Copyright 2016 Hewlett Packard Enterprise Development Company LP
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from pyspark.sql import SQLContext
|
||||||
|
|
||||||
|
from monasca_transform.component import Component
|
||||||
|
from monasca_transform.component.setter.rollup_quantity import RollupQuantity
|
||||||
|
from monasca_transform.component.usage.fetch_quantity import FetchQuantity
|
||||||
|
from monasca_transform.component.usage import UsageComponent
|
||||||
|
from monasca_transform.transform.transform_utils import InstanceUsageUtils
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
class CalculateRateException(Exception):
|
||||||
|
"""Exception thrown when calculating rate
|
||||||
|
Attributes:
|
||||||
|
value: string representing the error
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, value):
|
||||||
|
self.value = value
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return repr(self.value)
|
||||||
|
|
||||||
|
|
||||||
|
class CalculateRate(UsageComponent):
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def usage(transform_context, record_store_df):
|
||||||
|
"""component which groups together record store records by
|
||||||
|
provided group by columns list,sorts within the group by event
|
||||||
|
timestamp field, calculates the rate of change between the
|
||||||
|
oldest and latest values, and returns the resultant value as an
|
||||||
|
instance usage dataframe
|
||||||
|
"""
|
||||||
|
instance_usage_data_json_list = []
|
||||||
|
|
||||||
|
transform_spec_df = transform_context.transform_spec_df_info
|
||||||
|
|
||||||
|
# get aggregated metric name
|
||||||
|
agg_params = transform_spec_df.select(
|
||||||
|
"aggregation_params_map.aggregated_metric_name"). \
|
||||||
|
collect()[0].asDict()
|
||||||
|
aggregated_metric_name = agg_params["aggregated_metric_name"]
|
||||||
|
|
||||||
|
# get aggregation period
|
||||||
|
agg_params = transform_spec_df.select(
|
||||||
|
"aggregation_params_map.aggregation_period").collect()[0].asDict()
|
||||||
|
aggregation_period = agg_params["aggregation_period"]
|
||||||
|
|
||||||
|
# Fetch the oldest quantities
|
||||||
|
latest_instance_usage_df = \
|
||||||
|
FetchQuantity().usage_by_operation(transform_context,
|
||||||
|
record_store_df,
|
||||||
|
"avg")
|
||||||
|
|
||||||
|
# Roll up the latest quantities
|
||||||
|
latest_rolled_up_instance_usage_df = \
|
||||||
|
RollupQuantity().setter_by_operation(transform_context,
|
||||||
|
latest_instance_usage_df,
|
||||||
|
"sum")
|
||||||
|
|
||||||
|
# Fetch the oldest quantities
|
||||||
|
oldest_instance_usage_df = \
|
||||||
|
FetchQuantity().usage_by_operation(transform_context,
|
||||||
|
record_store_df,
|
||||||
|
"oldest")
|
||||||
|
|
||||||
|
# Roll up the oldest quantities
|
||||||
|
oldest_rolled_up_instance_usage_df = \
|
||||||
|
RollupQuantity().setter_by_operation(transform_context,
|
||||||
|
oldest_instance_usage_df,
|
||||||
|
"sum")
|
||||||
|
|
||||||
|
# Calculate the rate change by percentage
|
||||||
|
oldest_dict = oldest_rolled_up_instance_usage_df.collect()[0].asDict()
|
||||||
|
oldest_quantity = oldest_dict['quantity']
|
||||||
|
|
||||||
|
latest_dict = latest_rolled_up_instance_usage_df.collect()[0].asDict()
|
||||||
|
latest_quantity = latest_dict['quantity']
|
||||||
|
|
||||||
|
rate_percentage = \
|
||||||
|
((latest_quantity - oldest_quantity) / oldest_quantity) * 100
|
||||||
|
|
||||||
|
# create a new instance usage dict
|
||||||
|
instance_usage_dict = {"tenant_id": "all",
|
||||||
|
"user_id": "all",
|
||||||
|
"resource_uuid": "all",
|
||||||
|
"geolocation": "all",
|
||||||
|
"region": "all",
|
||||||
|
"zone": "all",
|
||||||
|
"host": "all",
|
||||||
|
"project_id": "all",
|
||||||
|
"aggregated_metric_name":
|
||||||
|
aggregated_metric_name,
|
||||||
|
"quantity": rate_percentage,
|
||||||
|
"firstrecord_timestamp_unix":
|
||||||
|
oldest_dict["firstrecord_timestamp_unix"],
|
||||||
|
"firstrecord_timestamp_string":
|
||||||
|
oldest_dict["firstrecord_timestamp_string"],
|
||||||
|
"lastrecord_timestamp_unix":
|
||||||
|
latest_dict["lastrecord_timestamp_unix"],
|
||||||
|
"lastrecord_timestamp_string":
|
||||||
|
latest_dict["lastrecord_timestamp_string"],
|
||||||
|
"record_count": oldest_dict["record_count"] +
|
||||||
|
latest_dict["record_count"],
|
||||||
|
"service_group":
|
||||||
|
Component.DEFAULT_UNAVAILABLE_VALUE,
|
||||||
|
"service_id":
|
||||||
|
Component.DEFAULT_UNAVAILABLE_VALUE,
|
||||||
|
"usage_date": latest_dict["usage_date"],
|
||||||
|
"usage_hour": latest_dict["usage_hour"],
|
||||||
|
"usage_minute": latest_dict["usage_minute"],
|
||||||
|
"aggregation_period": aggregation_period,
|
||||||
|
"processing_meta":
|
||||||
|
{"event_type":
|
||||||
|
latest_dict.get("event_type",
|
||||||
|
Component.
|
||||||
|
DEFAULT_UNAVAILABLE_VALUE)}
|
||||||
|
}
|
||||||
|
|
||||||
|
instance_usage_data_json = json.dumps(instance_usage_dict)
|
||||||
|
instance_usage_data_json_list.append(instance_usage_data_json)
|
||||||
|
spark_context = record_store_df.rdd.context
|
||||||
|
|
||||||
|
instance_usage_rdd = \
|
||||||
|
spark_context.parallelize(instance_usage_data_json_list)
|
||||||
|
|
||||||
|
sql_context = SQLContext\
|
||||||
|
.getOrCreate(record_store_df.rdd.context)
|
||||||
|
instance_usage_df = InstanceUsageUtils.create_df_from_json_rdd(
|
||||||
|
sql_context,
|
||||||
|
instance_usage_rdd)
|
||||||
|
|
||||||
|
return instance_usage_df
|
|
@ -299,6 +299,21 @@ class FetchQuantity(UsageComponent):
|
||||||
collect()[0].asDict()
|
collect()[0].asDict()
|
||||||
usage_fetch_operation = agg_params["usage_fetch_operation"]
|
usage_fetch_operation = agg_params["usage_fetch_operation"]
|
||||||
|
|
||||||
|
instance_usage_df = FetchQuantity.usage_by_operation(
|
||||||
|
transform_context, record_store_df, usage_fetch_operation)
|
||||||
|
|
||||||
|
return instance_usage_df
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def usage_by_operation(transform_context, record_store_df,
|
||||||
|
usage_fetch_operation):
|
||||||
|
"""component which groups together record store records by
|
||||||
|
provided group by columns list , sorts within the group by event
|
||||||
|
timestamp field, applies group stats udf and returns the latest
|
||||||
|
quantity as a instance usage dataframe
|
||||||
|
"""
|
||||||
|
transform_spec_df = transform_context.transform_spec_df_info
|
||||||
|
|
||||||
# check if operation is valid
|
# check if operation is valid
|
||||||
if not FetchQuantity. \
|
if not FetchQuantity. \
|
||||||
_is_valid_fetch_operation(usage_fetch_operation):
|
_is_valid_fetch_operation(usage_fetch_operation):
|
||||||
|
@ -323,8 +338,8 @@ class FetchQuantity(UsageComponent):
|
||||||
aggregation_group_by_list
|
aggregation_group_by_list
|
||||||
|
|
||||||
instance_usage_json_rdd = None
|
instance_usage_json_rdd = None
|
||||||
if (usage_fetch_operation == "latest"
|
if (usage_fetch_operation == "latest" or
|
||||||
or usage_fetch_operation == "oldest"):
|
usage_fetch_operation == "oldest"):
|
||||||
|
|
||||||
grouped_rows_rdd = None
|
grouped_rows_rdd = None
|
||||||
|
|
||||||
|
@ -367,10 +382,10 @@ class FetchQuantity(UsageComponent):
|
||||||
|
|
||||||
record_store_df_int = \
|
record_store_df_int = \
|
||||||
record_store_df.select(
|
record_store_df.select(
|
||||||
record_store_df.event_timestamp_unix
|
record_store_df.event_timestamp_unix.alias(
|
||||||
.alias("event_timestamp_unix_for_min"),
|
"event_timestamp_unix_for_min"),
|
||||||
record_store_df.event_timestamp_unix
|
record_store_df.event_timestamp_unix.alias(
|
||||||
.alias("event_timestamp_unix_for_max"),
|
"event_timestamp_unix_for_max"),
|
||||||
"*")
|
"*")
|
||||||
|
|
||||||
# for standard sum, max, min, avg operations on grouped data
|
# for standard sum, max, min, avg operations on grouped data
|
||||||
|
|
|
@ -251,10 +251,10 @@ class FetchQuantityUtil(UsageComponent):
|
||||||
#
|
#
|
||||||
quant_idle_perc_calc_df = quant_idle_perc_df.select(
|
quant_idle_perc_calc_df = quant_idle_perc_df.select(
|
||||||
col("quantity_df_alias.*"),
|
col("quantity_df_alias.*"),
|
||||||
when(col("idle_perc_df_alias.quantity")
|
when(col("idle_perc_df_alias.quantity") != 0.0,
|
||||||
!= 0.0,
|
ceil(((100.0 - col(
|
||||||
ceil(((100.0 - col("idle_perc_df_alias.quantity")))
|
"idle_perc_df_alias.quantity"))) * col(
|
||||||
* col("quantity_df_alias.quantity") / 100.0))
|
"quantity_df_alias.quantity") / 100.0))
|
||||||
.otherwise(col("quantity_df_alias.quantity"))
|
.otherwise(col("quantity_df_alias.quantity"))
|
||||||
.alias("utilized_quantity"),
|
.alias("utilized_quantity"),
|
||||||
|
|
||||||
|
|
|
@ -5,9 +5,10 @@
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_space_mb","metric_id_list":["disk_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_space_mb","metric_id_list":["disk_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_used_space_mb","metric_id_list":["disk_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_used_space_mb","metric_id_list":["disk_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.disk.total_allocated_gb","metric_id_list":["nova_disk_total_allocated_gb_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.disk.total_allocated_gb","metric_id_list":["nova_disk_total_allocated_gb_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.disk.allocation","metric_id_list":["vm_disk_allocation_all","vm_disk_allocation_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.disk.allocation","metric_id_list":["vm_disk_allocation_all","vm_disk_allocation_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"}
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.total_logical_cores","metric_id_list":["cpu_total_all","cpu_total_host","cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.total_logical_cores","metric_id_list":["cpu_total_all","cpu_total_host","cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.idle_perc","metric_id_list":["cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.idle_perc","metric_id_list":["cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id","resource_id"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id","resource_id"],"service_id":"host_metrics"}
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"}
|
||||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"}
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all","swift_usage_rate","swift_usage_host"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"}
|
||||||
|
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.avail","metric_id_list":["swift_avail_all","swift_avail_host"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"}
|
||||||
|
|
|
@ -17,3 +17,7 @@
|
||||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"}
|
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"}
|
||||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"}
|
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"}
|
||||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_all"}
|
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_all"}
|
||||||
|
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_host"}
|
||||||
|
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"calculate_rate","setters":["set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.rate_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"setter_rollup_group_by_list": [],"dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_rate","metric_id":"swift_usage_rate"}
|
||||||
|
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_avail_all","metric_id":"swift_avail_all"}
|
||||||
|
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_avail_all","metric_id":"swift_avail_host"}
|
||||||
|
|
|
@ -20,6 +20,7 @@ packages =
|
||||||
|
|
||||||
[entry_points]
|
[entry_points]
|
||||||
monasca_transform.usage =
|
monasca_transform.usage =
|
||||||
|
calculate_rate = monasca_transform.component.usage.calculate_rate:CalculateRate
|
||||||
fetch_quantity = monasca_transform.component.usage.fetch_quantity:FetchQuantity
|
fetch_quantity = monasca_transform.component.usage.fetch_quantity:FetchQuantity
|
||||||
fetch_quantity_util = monasca_transform.component.usage.fetch_quantity_util:FetchQuantityUtil
|
fetch_quantity_util = monasca_transform.component.usage.fetch_quantity_util:FetchQuantityUtil
|
||||||
|
|
||||||
|
|
|
@ -79,6 +79,14 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
|
||||||
metric_id='swift_usage_all',
|
metric_id='swift_usage_all',
|
||||||
expected_agg_metric_name='swiftlm.diskusage.val.size_agg',
|
expected_agg_metric_name='swiftlm.diskusage.val.size_agg',
|
||||||
transform_specs_dataframe=transform_specs_data_frame)
|
transform_specs_dataframe=transform_specs_data_frame)
|
||||||
|
self.check_metric(
|
||||||
|
metric_id='swift_avail_all',
|
||||||
|
expected_agg_metric_name='swiftlm.diskusage.val.avail_agg',
|
||||||
|
transform_specs_dataframe=transform_specs_data_frame)
|
||||||
|
self.check_metric(
|
||||||
|
metric_id='swift_usage_rate',
|
||||||
|
expected_agg_metric_name='swiftlm.diskusage.rate_agg',
|
||||||
|
transform_specs_dataframe=transform_specs_data_frame)
|
||||||
|
|
||||||
def check_metric(self, metric_id=None, expected_agg_metric_name=None,
|
def check_metric(self, metric_id=None, expected_agg_metric_name=None,
|
||||||
transform_specs_dataframe=None):
|
transform_specs_dataframe=None):
|
||||||
|
@ -107,7 +115,8 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
|
||||||
u'vm.mem.total_mb', u'vm.mem.used_mb',
|
u'vm.mem.total_mb', u'vm.mem.used_mb',
|
||||||
u'nova.vm.disk.total_allocated_gb',
|
u'nova.vm.disk.total_allocated_gb',
|
||||||
u'vm.disk.allocation', u'vm.cpu.utilization_perc',
|
u'vm.disk.allocation', u'vm.cpu.utilization_perc',
|
||||||
u'swiftlm.diskusage.host.val.size']),
|
u'swiftlm.diskusage.host.val.size',
|
||||||
|
u'swiftlm.diskusage.host.val.avail']),
|
||||||
Counter([row.event_type for row in
|
Counter([row.event_type for row in
|
||||||
pre_transform_specs_data_frame.collect()]))
|
pre_transform_specs_data_frame.collect()]))
|
||||||
|
|
||||||
|
@ -295,7 +304,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
|
||||||
self.check_list_field_for_row(
|
self.check_list_field_for_row(
|
||||||
row=vm_disk_allocation_all_row,
|
row=vm_disk_allocation_all_row,
|
||||||
field_name='required_raw_fields_list',
|
field_name='required_raw_fields_list',
|
||||||
expected_list=['creation_time', 'tenantId', 'resource_id'],
|
expected_list=['creation_time', 'tenant_id', 'resource_id'],
|
||||||
)
|
)
|
||||||
self.check_dict_field_for_row(
|
self.check_dict_field_for_row(
|
||||||
row=vm_disk_allocation_all_row,
|
row=vm_disk_allocation_all_row,
|
||||||
|
@ -346,7 +355,8 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
|
||||||
self.check_list_field_for_row(
|
self.check_list_field_for_row(
|
||||||
row=swiftlm_diskusage_all_row,
|
row=swiftlm_diskusage_all_row,
|
||||||
field_name='metric_id_list',
|
field_name='metric_id_list',
|
||||||
expected_list=['swift_usage_all']
|
expected_list=['swift_usage_all', 'swift_usage_host',
|
||||||
|
'swift_usage_rate']
|
||||||
)
|
)
|
||||||
self.check_list_field_for_row(
|
self.check_list_field_for_row(
|
||||||
row=swiftlm_diskusage_all_row,
|
row=swiftlm_diskusage_all_row,
|
||||||
|
@ -367,6 +377,35 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
|
||||||
expected_value='host_metrics'
|
expected_value='host_metrics'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# swiftlm.diskusage.host.val.avail
|
||||||
|
event_type = 'swiftlm.diskusage.host.val.avail'
|
||||||
|
swiftlm_diskavail_all_row = self.get_row_for_event_type(
|
||||||
|
event_type=event_type,
|
||||||
|
pre_transform_specs_data_frame=pre_transform_specs_data_frame)
|
||||||
|
self.check_list_field_for_row(
|
||||||
|
row=swiftlm_diskavail_all_row,
|
||||||
|
field_name='metric_id_list',
|
||||||
|
expected_list=['swift_avail_all', 'swift_avail_host']
|
||||||
|
)
|
||||||
|
self.check_list_field_for_row(
|
||||||
|
row=swiftlm_diskavail_all_row,
|
||||||
|
field_name='required_raw_fields_list',
|
||||||
|
expected_list=['creation_time', 'hostname', 'mount',
|
||||||
|
'device'],
|
||||||
|
)
|
||||||
|
self.check_dict_field_for_row(
|
||||||
|
row=swiftlm_diskavail_all_row,
|
||||||
|
field_name='event_processing_params',
|
||||||
|
expected_dict={
|
||||||
|
"set_default_zone_to": "1",
|
||||||
|
"set_default_geolocation_to": "1",
|
||||||
|
"set_default_region_to": "W"})
|
||||||
|
self.check_value_field_for_row(
|
||||||
|
row=swiftlm_diskavail_all_row,
|
||||||
|
field_name='service_id',
|
||||||
|
expected_value='host_metrics'
|
||||||
|
)
|
||||||
|
|
||||||
def get_row_for_event_type(self,
|
def get_row_for_event_type(self,
|
||||||
event_type=None,
|
event_type=None,
|
||||||
pre_transform_specs_data_frame=None):
|
pre_transform_specs_data_frame=None):
|
||||||
|
|
|
@ -114,7 +114,7 @@ class SparkTest(SparkContextTest):
|
||||||
result = simple_count_transform(rdd_monasca_with_offsets)
|
result = simple_count_transform(rdd_monasca_with_offsets)
|
||||||
|
|
||||||
# Verify it worked
|
# Verify it worked
|
||||||
self.assertEqual(result, 347)
|
self.assertEqual(result, 363)
|
||||||
|
|
||||||
# Call the primary method in mon_metrics_kafka
|
# Call the primary method in mon_metrics_kafka
|
||||||
MonMetricsKafkaProcessor.rdd_to_recordstore(
|
MonMetricsKafkaProcessor.rdd_to_recordstore(
|
||||||
|
@ -972,13 +972,14 @@ class SparkTest(SparkContextTest):
|
||||||
.get('metric').get('value_meta')
|
.get('metric').get('value_meta')
|
||||||
.get('lastrecord_timestamp'))
|
.get('lastrecord_timestamp'))
|
||||||
|
|
||||||
# Verify swiftlm.diskusage.val.size_agg
|
# Verify swiftlm.diskusage.val.size_agg for all hosts
|
||||||
used_swift_agg_metric = [
|
used_swift_agg_metric = [
|
||||||
value for value in metrics
|
value for value in metrics
|
||||||
if value.get('metric').get('name') ==
|
if value.get('metric').get('name') ==
|
||||||
'swiftlm.diskusage.val.size_agg'][0]
|
'swiftlm.diskusage.val.size_agg' and
|
||||||
|
value.get('metric').get('dimensions').get('host') == 'all'][0]
|
||||||
|
|
||||||
self.assertEqual(360000000000342.0,
|
self.assertEqual(5291.0,
|
||||||
used_swift_agg_metric.get('metric').get('value'))
|
used_swift_agg_metric.get('metric').get('value'))
|
||||||
self.assertEqual('useast',
|
self.assertEqual('useast',
|
||||||
used_swift_agg_metric.get('meta').get('region'))
|
used_swift_agg_metric.get('meta').get('region'))
|
||||||
|
@ -986,23 +987,196 @@ class SparkTest(SparkContextTest):
|
||||||
used_swift_agg_metric.get('meta').get('tenantId'))
|
used_swift_agg_metric.get('meta').get('tenantId'))
|
||||||
self.assertEqual('all',
|
self.assertEqual('all',
|
||||||
used_swift_agg_metric.get('metric').get('dimensions')
|
used_swift_agg_metric.get('metric').get('dimensions')
|
||||||
.get('host'))
|
.get('project_id'))
|
||||||
|
self.assertEqual('hourly',
|
||||||
|
used_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('aggregation_period'))
|
||||||
|
self.assertEqual(17.0,
|
||||||
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('record_count'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('firstrecord_timestamp'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('lastrecord_timestamp'))
|
||||||
|
|
||||||
|
# Verify swiftlm.diskusage.val.size_agg for host a
|
||||||
|
used_swift_agg_metric = [
|
||||||
|
value for value in metrics
|
||||||
|
if value.get('metric').get('name') ==
|
||||||
|
'swiftlm.diskusage.val.size_agg' and
|
||||||
|
value.get('metric').get('dimensions').get('host') == 'a'][0]
|
||||||
|
|
||||||
|
self.assertEqual(2848.0,
|
||||||
|
used_swift_agg_metric.get('metric').get('value'))
|
||||||
|
self.assertEqual('useast',
|
||||||
|
used_swift_agg_metric.get('meta').get('region'))
|
||||||
|
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
|
||||||
|
used_swift_agg_metric.get('meta').get('tenantId'))
|
||||||
self.assertEqual('all',
|
self.assertEqual('all',
|
||||||
used_swift_agg_metric.get('metric').get('dimensions')
|
used_swift_agg_metric.get('metric').get('dimensions')
|
||||||
.get('project_id'))
|
.get('project_id'))
|
||||||
self.assertEqual('hourly',
|
self.assertEqual('hourly',
|
||||||
used_swift_agg_metric.get('metric').get('dimensions')
|
used_swift_agg_metric.get('metric').get('dimensions')
|
||||||
.get('aggregation_period'))
|
.get('aggregation_period'))
|
||||||
self.assertEqual(18.0,
|
self.assertEqual(8.0,
|
||||||
used_swift_agg_metric.get('metric').get('value_meta')
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
.get('record_count'))
|
.get('record_count'))
|
||||||
self.assertEqual('2016-06-10 20:27:01',
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
used_swift_agg_metric.get('metric').get('value_meta')
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
.get('firstrecord_timestamp'))
|
.get('firstrecord_timestamp'))
|
||||||
self.assertEqual('2016-06-10 20:27:20',
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
used_swift_agg_metric.get('metric').get('value_meta')
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
.get('lastrecord_timestamp'))
|
.get('lastrecord_timestamp'))
|
||||||
|
|
||||||
|
# Verify swiftlm.diskusage.val.size_agg for host b
|
||||||
|
used_swift_agg_metric = [
|
||||||
|
value for value in metrics
|
||||||
|
if value.get('metric').get('name') ==
|
||||||
|
'swiftlm.diskusage.val.size_agg' and
|
||||||
|
value.get('metric').get('dimensions').get('host') == 'b'][0]
|
||||||
|
|
||||||
|
self.assertEqual(2443.0,
|
||||||
|
used_swift_agg_metric.get('metric').get('value'))
|
||||||
|
self.assertEqual('useast',
|
||||||
|
used_swift_agg_metric.get('meta').get('region'))
|
||||||
|
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
|
||||||
|
used_swift_agg_metric.get('meta').get('tenantId'))
|
||||||
|
self.assertEqual('all',
|
||||||
|
used_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('project_id'))
|
||||||
|
self.assertEqual('hourly',
|
||||||
|
used_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('aggregation_period'))
|
||||||
|
self.assertEqual(9.0,
|
||||||
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('record_count'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('firstrecord_timestamp'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
used_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('lastrecord_timestamp'))
|
||||||
|
|
||||||
|
# Verify swiftlm.diskusage.val.avail_agg for all hosts
|
||||||
|
avail_swift_agg_metric = [
|
||||||
|
value for value in metrics
|
||||||
|
if value.get('metric').get('name') ==
|
||||||
|
'swiftlm.diskusage.val.avail_agg' and
|
||||||
|
value.get('metric').get('dimensions').get('host') == 'all'][0]
|
||||||
|
|
||||||
|
self.assertEqual(5291.0,
|
||||||
|
avail_swift_agg_metric.get('metric').get('value'))
|
||||||
|
self.assertEqual('useast',
|
||||||
|
avail_swift_agg_metric.get('meta').get('region'))
|
||||||
|
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
|
||||||
|
avail_swift_agg_metric.get('meta').get('tenantId'))
|
||||||
|
self.assertEqual('all',
|
||||||
|
avail_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('project_id'))
|
||||||
|
self.assertEqual('hourly',
|
||||||
|
avail_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('aggregation_period'))
|
||||||
|
self.assertEqual(17.0,
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('record_count'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('firstrecord_timestamp'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('lastrecord_timestamp'))
|
||||||
|
|
||||||
|
# Verify swiftlm.diskusage.val.avail_agg for host a
|
||||||
|
avail_swift_agg_metric = [
|
||||||
|
value for value in metrics
|
||||||
|
if value.get('metric').get('name') ==
|
||||||
|
'swiftlm.diskusage.val.avail_agg' and
|
||||||
|
value.get('metric').get('dimensions').get('host') == 'a'][0]
|
||||||
|
|
||||||
|
self.assertEqual(2848.0,
|
||||||
|
avail_swift_agg_metric.get('metric').get('value'))
|
||||||
|
self.assertEqual('useast',
|
||||||
|
avail_swift_agg_metric.get('meta').get('region'))
|
||||||
|
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
|
||||||
|
avail_swift_agg_metric.get('meta').get('tenantId'))
|
||||||
|
self.assertEqual('all',
|
||||||
|
avail_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('project_id'))
|
||||||
|
self.assertEqual('hourly',
|
||||||
|
avail_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('aggregation_period'))
|
||||||
|
self.assertEqual(8.0,
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('record_count'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('firstrecord_timestamp'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('lastrecord_timestamp'))
|
||||||
|
|
||||||
|
# Verify swiftlm.diskusage.val.avail_agg for host b
|
||||||
|
avail_swift_agg_metric = [
|
||||||
|
value for value in metrics
|
||||||
|
if value.get('metric').get('name') ==
|
||||||
|
'swiftlm.diskusage.val.avail_agg' and
|
||||||
|
value.get('metric').get('dimensions').get('host') == 'b'][0]
|
||||||
|
|
||||||
|
self.assertEqual(2443.0,
|
||||||
|
avail_swift_agg_metric.get('metric').get('value'))
|
||||||
|
self.assertEqual('useast',
|
||||||
|
avail_swift_agg_metric.get('meta').get('region'))
|
||||||
|
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
|
||||||
|
avail_swift_agg_metric.get('meta').get('tenantId'))
|
||||||
|
self.assertEqual('all',
|
||||||
|
avail_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('project_id'))
|
||||||
|
self.assertEqual('hourly',
|
||||||
|
avail_swift_agg_metric.get('metric').get('dimensions')
|
||||||
|
.get('aggregation_period'))
|
||||||
|
self.assertEqual(9.0,
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('record_count'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('firstrecord_timestamp'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
avail_swift_agg_metric.get('metric').get('value_meta')
|
||||||
|
.get('lastrecord_timestamp'))
|
||||||
|
|
||||||
|
# Verify swiftlm.diskusage.rate_agg metrics
|
||||||
|
diskusage_rate_agg_metric = [
|
||||||
|
value for value in metrics
|
||||||
|
if value.get('metric').get('name') ==
|
||||||
|
'swiftlm.diskusage.rate_agg'][0]
|
||||||
|
|
||||||
|
self.assertEqual(15.650273224043715,
|
||||||
|
diskusage_rate_agg_metric.get('metric').get('value'))
|
||||||
|
self.assertEqual('useast',
|
||||||
|
diskusage_rate_agg_metric.get('meta').get('region'))
|
||||||
|
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
|
||||||
|
diskusage_rate_agg_metric.get('meta').get('tenantId'))
|
||||||
|
self.assertEqual('all',
|
||||||
|
diskusage_rate_agg_metric.get('metric')
|
||||||
|
.get('dimensions').get('host'))
|
||||||
|
self.assertEqual('all',
|
||||||
|
diskusage_rate_agg_metric.get('metric')
|
||||||
|
.get('dimensions').get('project_id'))
|
||||||
|
self.assertEqual('hourly',
|
||||||
|
diskusage_rate_agg_metric.get('metric')
|
||||||
|
.get('dimensions').get('aggregation_period'))
|
||||||
|
self.assertEqual(34.0,
|
||||||
|
diskusage_rate_agg_metric.get('metric')
|
||||||
|
.get('value_meta').get('record_count'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
diskusage_rate_agg_metric.get('metric')
|
||||||
|
.get('value_meta').get('firstrecord_timestamp'))
|
||||||
|
self.assertEqual('2016-06-10 20:27:01',
|
||||||
|
diskusage_rate_agg_metric.get('metric')
|
||||||
|
.get('value_meta').get('lastrecord_timestamp'))
|
||||||
|
|
||||||
|
|
||||||
def simple_count_transform(rdd):
|
def simple_count_transform(rdd):
|
||||||
return rdd.count()
|
return rdd.count()
|
||||||
|
|
|
@ -296,15 +296,15 @@
|
||||||
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0006-mgmt","cluster":"compute"},"timestamp":1463498061000,"value":82.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498062}')
|
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0006-mgmt","cluster":"compute"},"timestamp":1463498061000,"value":82.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498062}')
|
||||||
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0003-mgmt","cluster":"compute"},"timestamp":1463498078000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498080}')
|
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0003-mgmt","cluster":"compute"},"timestamp":1463498078000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498080}')
|
||||||
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0002-mgmt","cluster":"compute"},"timestamp":1463498084000,"value":132.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498086}')
|
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0002-mgmt","cluster":"compute"},"timestamp":1463498084000,"value":132.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498086}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"c","component":"vm","service":"compute","hostname":"dummy1","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":28.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"c","component":"vm","service":"compute","hostname":"dummy1","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":28.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"d","component":"vm","service":"compute","hostname":"dummy2","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":30.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"d","component":"vm","service":"compute","hostname":"dummy2","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":30.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"e","component":"vm","service":"compute","hostname":"dummy3","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":32.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"e","component":"vm","service":"compute","hostname":"dummy3","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":32.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
|
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
|
||||||
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"7b805207-3c97-4ef9-8b28-3fac171f2a96","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283861000,"value":5.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
|
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"7b805207-3c97-4ef9-8b28-3fac171f2a96","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283861000,"value":5.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
|
||||||
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"a07858d1-aa13-4259-9873-f23170bc9e1b","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283895000,"value":35.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
|
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"a07858d1-aa13-4259-9873-f23170bc9e1b","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283895000,"value":35.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
|
||||||
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"4cd09eac-854d-49eb-993d-914271f8e1fe","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283896000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
|
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"4cd09eac-854d-49eb-993d-914271f8e1fe","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283896000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
|
||||||
|
@ -327,21 +327,37 @@
|
||||||
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"b","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316994000,"value":5120.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316995}')
|
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"b","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316994000,"value":5120.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316995}')
|
||||||
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"c","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317024000,"value":2048.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317025}')
|
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"c","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317024000,"value":2048.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317025}')
|
||||||
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"d","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317054000,"value":6144.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317055}')
|
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"d","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317054000,"value":6144.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317055}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":904},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000006},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":804},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000008},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":928},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000010},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000012},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000014},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421004,"value":1128},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000016},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421001,"value":916},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000018},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":901},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000020},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000022},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":602},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000024},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":926},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1126},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000030},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000032},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000034},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000036},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":904},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":804},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":928},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421004,"value":1128},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421001,"value":916},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":901},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":602},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":926},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1126},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
|
||||||
|
|
|
@ -25,6 +25,8 @@ from monasca_transform.component.setter.set_aggregated_metric_name \
|
||||||
import SetAggregatedMetricName
|
import SetAggregatedMetricName
|
||||||
from monasca_transform.component.setter.set_aggregated_period \
|
from monasca_transform.component.setter.set_aggregated_period \
|
||||||
import SetAggregatedPeriod
|
import SetAggregatedPeriod
|
||||||
|
from monasca_transform.component.usage.calculate_rate \
|
||||||
|
import CalculateRate
|
||||||
from monasca_transform.component.usage.fetch_quantity \
|
from monasca_transform.component.usage.fetch_quantity \
|
||||||
import FetchQuantity
|
import FetchQuantity
|
||||||
from monasca_transform.component.usage.fetch_quantity_util \
|
from monasca_transform.component.usage.fetch_quantity_util \
|
||||||
|
@ -49,6 +51,13 @@ class MockComponentManager(object):
|
||||||
'FetchQuantityUtil',
|
'FetchQuantityUtil',
|
||||||
FetchQuantityUtil(),
|
FetchQuantityUtil(),
|
||||||
None),
|
None),
|
||||||
|
Extension(
|
||||||
|
'calculate_rate',
|
||||||
|
'monasca_transform.component.usage.'
|
||||||
|
'calculate_rate:'
|
||||||
|
'CalculateRate',
|
||||||
|
CalculateRate(),
|
||||||
|
None),
|
||||||
])
|
])
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
@ -138,12 +138,6 @@ class TestVmCpuAllocatedAgg(SparkContextTest):
|
||||||
rdd_monasca_with_offsets = rdd_monasca.map(
|
rdd_monasca_with_offsets = rdd_monasca.map(
|
||||||
lambda x: RddTransformContext(x, transform_context))
|
lambda x: RddTransformContext(x, transform_context))
|
||||||
|
|
||||||
# Do something simple with the RDD
|
|
||||||
result = simple_count_transform(rdd_monasca_with_offsets)
|
|
||||||
|
|
||||||
# Verify it worked
|
|
||||||
self.assertEqual(result, 347)
|
|
||||||
|
|
||||||
# Call the primary method in mon_metrics_kafka
|
# Call the primary method in mon_metrics_kafka
|
||||||
MonMetricsKafkaProcessor.rdd_to_recordstore(
|
MonMetricsKafkaProcessor.rdd_to_recordstore(
|
||||||
rdd_monasca_with_offsets)
|
rdd_monasca_with_offsets)
|
||||||
|
@ -317,12 +311,6 @@ class TestVmCpuAllocatedAgg(SparkContextTest):
|
||||||
rdd_monasca_with_offsets = rdd_monasca.map(
|
rdd_monasca_with_offsets = rdd_monasca.map(
|
||||||
lambda x: RddTransformContext(x, transform_context))
|
lambda x: RddTransformContext(x, transform_context))
|
||||||
|
|
||||||
# Do something simple with the RDD
|
|
||||||
result = simple_count_transform(rdd_monasca_with_offsets)
|
|
||||||
|
|
||||||
# Verify it worked
|
|
||||||
self.assertEqual(result, 347)
|
|
||||||
|
|
||||||
# Call the primary method in mon_metrics_kafka
|
# Call the primary method in mon_metrics_kafka
|
||||||
MonMetricsKafkaProcessor.rdd_to_recordstore(
|
MonMetricsKafkaProcessor.rdd_to_recordstore(
|
||||||
rdd_monasca_with_offsets)
|
rdd_monasca_with_offsets)
|
||||||
|
|
Loading…
Reference in New Issue