# Copyright 2022 Red Hat # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # This prometheus-proxy is intended to abstract the prometheus metrics # exported from the reference provider driver load balancing engines (haproxy # and lvs) such that all of the provider drivers can expose a consistent set # of metrics. It also aligns the terms to be consistent with Octavia # terminology. from http.server import SimpleHTTPRequestHandler from http.server import ThreadingHTTPServer import os import signal import sys import threading import time import traceback import urllib.request import psutil from octavia.amphorae.backends.utils import network_namespace from octavia.common import constants as consts METRICS_URL = "http://127.0.0.1:9101/metrics" PRINT_REJECTED = False EXIT_EVENT = threading.Event() # A dictionary of prometheus metrics mappings. # Key: The metric string to match # Value: A tuple of replacement data. # tuple[0]: The string to replace the key with. # tuple[1]: If not None, the replacement HELP line for the metric. # If tuple[1] is None, the key will be replaced in the HELP string. # tuple[2]: If not None, includes a dictionary of additional substitutions. # tuple[2] substitutions happen prior to the key replacement in tuple[0]. METRIC_MAP = { # Load balancer metrics "haproxy_process_pool_allocated_bytes ": ("octavia_memory_pool_allocated_bytes ", "# HELP octavia_memory_pool_allocated_bytes Total amount of memory " "allocated in the memory pools (in bytes).\n", None), "haproxy_process_pool_used_bytes ": ("octavia_memory_pool_used_bytes ", "# HELP octavia_memory_pool_used_bytes Total amount of memory used " "in the memory pools (in bytes).\n", None), "haproxy_process_pool_failures_total ": ("octavia_memory_pool_failures_total ", "# HELP octavia_memory_pool_failures_total Total number of failed " "memory pool allocations.\n", None), "haproxy_process_max_connections ": ("octavia_loadbalancer_max_connections ", None, None), "haproxy_process_current_connections ": ("octavia_loadbalancer_current_connections ", None, None), # TODO(johnsom) consider adding in UDP "haproxy_process_connections_total ": ("octavia_loadbalancer_connections_total ", None, None), # TODO(johnsom) consider adding in UDP (and update help string) "haproxy_process_requests_total ": ("octavia_loadbalancer_requests_total ", None, None), "haproxy_process_max_ssl_connections ": ("octavia_loadbalancer_max_ssl_connections ", None, None), "haproxy_process_current_ssl_connections ": ("octavia_loadbalancer_current_ssl_connections ", "# HELP octavia_loadbalancer_current_ssl_connections Number of " "active SSL connections.\n", None), "haproxy_process_ssl_connections_total ": ("octavia_loadbalancer_ssl_connections_total ", None, None), "haproxy_process_current_connection_rate ": ("octavia_loadbalancer_current_connection_rate ", None, None), "haproxy_process_limit_connection_rate ": ("octavia_loadbalancer_limit_connection_rate ", None, None), "haproxy_process_max_connection_rate ": ("octavia_loadbalancer_max_connection_rate ", None, None), "haproxy_process_current_session_rate ": ("octavia_loadbalancer_current_session_rate ", None, None), "haproxy_process_limit_session_rate ": ("octavia_loadbalancer_limit_session_rate ", None, None), "haproxy_process_max_session_rate ": ("octavia_loadbalancer_max_session_rate ", None, None), "haproxy_process_current_ssl_rate ": ("octavia_loadbalancer_current_ssl_rate ", None, None), "haproxy_process_limit_ssl_rate ": ("octavia_loadbalancer_limit_ssl_rate ", None, None), "haproxy_process_max_ssl_rate ": ("octavia_loadbalancer_max_ssl_rate ", None, None), "haproxy_process_current_frontend_ssl_key_rate ": ("octavia_loadbalancer_current_frontend_ssl_key_rate ", None, None), "haproxy_process_max_frontend_ssl_key_rate ": ("octavia_loadbalancer_max_frontend_ssl_key_rate ", None, None), "haproxy_process_frontend_ssl_reuse ": ("octavia_loadbalancer_frontend_ssl_reuse ", None, None), "haproxy_process_current_backend_ssl_key_rate ": ("octavia_loadbalancer_current_backend_ssl_key_rate ", None, None), "haproxy_process_max_backend_ssl_key_rate ": ("octavia_loadbalancer_max_backend_ssl_key_rate ", None, None), "haproxy_process_ssl_cache_lookups_total ": ("octavia_loadbalancer_ssl_cache_lookups_total ", None, None), "haproxy_process_ssl_cache_misses_total ": ("octavia_loadbalancer_ssl_cache_misses_total ", None, None), "haproxy_process_http_comp_bytes_in_total ": ("octavia_loadbalancer_http_comp_bytes_in_total ", None, None), "haproxy_process_http_comp_bytes_out_total ": ("octavia_loadbalancer_http_comp_bytes_out_total ", None, None), "haproxy_process_limit_http_comp ": ("octavia_loadbalancer_limit_http_comp ", None, None), "haproxy_process_listeners ": ("octavia_loadbalancer_listeners ", None, None), "haproxy_process_dropped_logs_total ": ("octavia_loadbalancer_dropped_logs_total ", None, None), # Listener metrics "haproxy_frontend_status ": ("octavia_listener_status ", "# HELP octavia_listener_status Current status of the listener.\n", None), "haproxy_frontend_status{": ("octavia_listener_status{", None, {"proxy=": "listener="}), "haproxy_frontend_current_sessions ": ("octavia_listener_current_sessions ", None, None), "haproxy_frontend_current_sessions{": ("octavia_listener_current_sessions{", None, {"proxy=": "listener="}), "haproxy_frontend_max_sessions ": ("octavia_listener_max_sessions ", None, None), "haproxy_frontend_max_sessions{": ("octavia_listener_max_sessions{", None, {"proxy=": "listener="}), "haproxy_frontend_limit_sessions ": ("octavia_listener_limit_sessions ", None, None), "haproxy_frontend_limit_sessions{": ("octavia_listener_limit_sessions{", None, {"proxy=": "listener="}), "haproxy_frontend_sessions_total ": ("octavia_listener_sessions_total ", None, None), "haproxy_frontend_sessions_total{": ("octavia_listener_sessions_total{", None, {"proxy=": "listener="}), "haproxy_frontend_limit_session_rate ": ("octavia_listener_limit_session_rate ", None, None), "haproxy_frontend_limit_session_rate{": ("octavia_listener_limit_session_rate{", None, {"proxy=": "listener="}), "haproxy_frontend_max_session_rate ": ("octavia_listener_max_session_rate ", None, None), "haproxy_frontend_max_session_rate{": ("octavia_listener_max_session_rate{", None, {"proxy=": "listener="}), "haproxy_frontend_connections_rate_max ": ("octavia_listener_connections_rate_max ", None, None), "haproxy_frontend_connections_rate_max{": ("octavia_listener_connections_rate_max{", None, {"proxy=": "listener="}), "haproxy_frontend_connections_total ": ("octavia_listener_connections_total ", None, None), "haproxy_frontend_connections_total{": ("octavia_listener_connections_total{", None, {"proxy=": "listener="}), "haproxy_frontend_bytes_in_total ": ("octavia_listener_bytes_in_total ", None, None), "haproxy_frontend_bytes_in_total{": ("octavia_listener_bytes_in_total{", None, {"proxy=": "listener="}), "haproxy_frontend_bytes_out_total ": ("octavia_listener_bytes_out_total ", None, None), "haproxy_frontend_bytes_out_total{": ("octavia_listener_bytes_out_total{", None, {"proxy=": "listener="}), "haproxy_frontend_requests_denied_total ": ("octavia_listener_requests_denied_total ", None, None), "haproxy_frontend_requests_denied_total{": ("octavia_listener_requests_denied_total{", None, {"proxy=": "listener="}), "haproxy_frontend_responses_denied_total ": ("octavia_listener_responses_denied_total ", None, None), "haproxy_frontend_responses_denied_total{": ("octavia_listener_responses_denied_total{", None, {"proxy=": "listener="}), "haproxy_frontend_request_errors_total ": ("octavia_listener_request_errors_total ", None, None), "haproxy_frontend_request_errors_total{": ("octavia_listener_request_errors_total{", None, {"proxy=": "listener="}), "haproxy_frontend_denied_connections_total ": ("octavia_listener_denied_connections_total ", "# HELP octavia_listener_denied_connections_total Total number of " "requests denied by connection rules.\n", None), "haproxy_frontend_denied_connections_total{": ("octavia_listener_denied_connections_total{", None, {"proxy=": "listener="}), "haproxy_frontend_denied_sessions_total ": ("octavia_listener_denied_sessions_total ", "# HELP octavia_listener_denied_sessions_total Total number of " "requests denied by session rules.\n", None), "haproxy_frontend_denied_sessions_total{": ("octavia_listener_denied_sessions_total{", None, {"proxy=": "listener="}), "haproxy_frontend_failed_header_rewriting_total ": ("octavia_listener_failed_header_rewriting_total ", "# HELP octavia_listener_failed_header_rewriting_total Total number " "of failed header rewriting rules.\n", None), "haproxy_frontend_failed_header_rewriting_total{": ("octavia_listener_failed_header_rewriting_total{", None, {"proxy=": "listener="}), "haproxy_frontend_http_requests_rate_max ": ("octavia_listener_http_requests_rate_max ", None, None), "haproxy_frontend_http_requests_rate_max{": ("octavia_listener_http_requests_rate_max{", None, {"proxy=": "listener="}), "haproxy_frontend_http_requests_total ": ("octavia_listener_http_requests_total ", None, None), "haproxy_frontend_http_requests_total{": ("octavia_listener_http_requests_total{", None, {"proxy=": "listener="}), "haproxy_frontend_http_responses_total ": ("octavia_listener_http_responses_total ", None, None), "haproxy_frontend_http_responses_total{": ("octavia_listener_http_responses_total{", None, {"proxy=": "listener="}), "haproxy_frontend_intercepted_requests_total ": ("octavia_listener_intercepted_requests_total ", None, None), "haproxy_frontend_intercepted_requests_total{": ("octavia_listener_intercepted_requests_total{", None, {"proxy=": "listener="}), "haproxy_frontend_http_cache_lookups_total ": ("octavia_listener_http_cache_lookups_total ", None, None), "haproxy_frontend_http_cache_lookups_total{": ("octavia_listener_http_cache_lookups_total{", None, {"proxy=": "listener="}), "haproxy_frontend_http_cache_hits_total ": ("octavia_listener_http_cache_hits_total ", None, None), "haproxy_frontend_http_cache_hits_total{": ("octavia_listener_http_cache_hits_total{", None, {"proxy=": "listener="}), "haproxy_frontend_http_comp_bytes_in_total ": ("octavia_listener_http_comp_bytes_in_total ", None, None), "haproxy_frontend_http_comp_bytes_in_total{": ("octavia_listener_http_comp_bytes_in_total{", None, {"proxy=": "listener="}), "haproxy_frontend_http_comp_bytes_out_total ": ("octavia_listener_http_comp_bytes_out_total ", None, None), "haproxy_frontend_http_comp_bytes_out_total{": ("octavia_listener_http_comp_bytes_out_total{", None, {"proxy=": "listener="}), "haproxy_frontend_http_comp_bytes_bypassed_total ": ("octavia_listener_http_comp_bytes_bypassed_total ", None, None), "haproxy_frontend_http_comp_bytes_bypassed_total{": ("octavia_listener_http_comp_bytes_bypassed_total{", None, {"proxy=": "listener="}), "haproxy_frontend_http_comp_responses_total ": ("octavia_listener_http_comp_responses_total ", None, None), "haproxy_frontend_http_comp_responses_total{": ("octavia_listener_http_comp_responses_total{", None, {"proxy=": "listener="}), # Pool Metrics "haproxy_backend_status ": ("octavia_pool_status ", "# HELP octavia_pool_status Current status of the pool.\n", None), "haproxy_backend_status{": ("octavia_pool_status{", None, {"proxy=": "pool="}), "haproxy_backend_current_sessions ": ("octavia_pool_current_sessions ", None, None), "haproxy_backend_current_sessions{": ("octavia_pool_current_sessions{", None, {"proxy=": "pool="}), "haproxy_backend_max_sessions ": ("octavia_pool_max_sessions ", None, None), "haproxy_backend_max_sessions{": ("octavia_pool_max_sessions{", None, {"proxy=": "pool="}), "haproxy_backend_limit_sessions ": ("octavia_pool_limit_sessions ", None, None), "haproxy_backend_limit_sessions{": ("octavia_pool_limit_sessions{", None, {"proxy=": "pool="}), "haproxy_backend_sessions_total ": ("octavia_pool_sessions_total ", None, None), "haproxy_backend_sessions_total{": ("octavia_pool_sessions_total{", None, {"proxy=": "pool="}), "haproxy_backend_max_session_rate ": ("octavia_pool_max_session_rate ", None, None), "haproxy_backend_max_session_rate{": ("octavia_pool_max_session_rate{", None, {"proxy=": "pool="}), "haproxy_backend_last_session_seconds ": ("octavia_pool_last_session_seconds ", "# HELP octavia_pool_last_session_seconds Number of seconds since " "last session assigned to a member.\n", None, None), "haproxy_backend_last_session_seconds{": ("octavia_pool_last_session_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_current_queue ": ("octavia_pool_current_queue ", None, None), "haproxy_backend_current_queue{": ("octavia_pool_current_queue{", None, {"proxy=": "pool="}), "haproxy_backend_max_queue ": ("octavia_pool_max_queue ", None, None), "haproxy_backend_max_queue{": ("octavia_pool_max_queue{", None, {"proxy=": "pool="}), "haproxy_backend_connection_attempts_total ": ("octavia_pool_connection_attempts_total ", None, None), "haproxy_backend_connection_attempts_total{": ("octavia_pool_connection_attempts_total{", None, {"proxy=": "pool="}), "haproxy_backend_connection_reuses_total ": ("octavia_pool_connection_reuses_total ", None, None), "haproxy_backend_connection_reuses_total{": ("octavia_pool_connection_reuses_total{", None, {"proxy=": "pool="}), "haproxy_backend_bytes_in_total ": ("octavia_pool_bytes_in_total ", None, None), "haproxy_backend_bytes_in_total{": ("octavia_pool_bytes_in_total{", None, {"proxy=": "pool="}), "haproxy_backend_bytes_out_total ": ("octavia_pool_bytes_out_total ", None, None), "haproxy_backend_bytes_out_total{": ("octavia_pool_bytes_out_total{", None, {"proxy=": "pool="}), "haproxy_backend_queue_time_average_seconds ": ("octavia_pool_queue_time_average_seconds ", None, None), "haproxy_backend_queue_time_average_seconds{": ("octavia_pool_queue_time_average_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_connect_time_average_seconds ": ("octavia_pool_connect_time_average_seconds ", None, None), "haproxy_backend_connect_time_average_seconds{": ("octavia_pool_connect_time_average_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_response_time_average_seconds ": ("octavia_pool_response_time_average_seconds ", None, None), "haproxy_backend_response_time_average_seconds{": ("octavia_pool_response_time_average_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_total_time_average_seconds ": ("octavia_pool_total_time_average_seconds ", None, None), "haproxy_backend_total_time_average_seconds{": ("octavia_pool_total_time_average_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_max_queue_time_seconds ": ("octavia_pool_max_queue_time_seconds ", None, None), "haproxy_backend_max_queue_time_seconds{": ("octavia_pool_max_queue_time_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_max_connect_time_seconds ": ("octavia_pool_max_connect_time_seconds ", None, None), "haproxy_backend_max_connect_time_seconds{": ("octavia_pool_max_connect_time_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_max_response_time_seconds ": ("octavia_pool_max_response_time_seconds ", "# HELP octavia_pool_max_response_time_seconds Maximum observed " "time spent waiting for a member response.\n", None), "haproxy_backend_max_response_time_seconds{": ("octavia_pool_max_response_time_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_max_total_time_seconds ": ("octavia_pool_max_total_time_seconds ", None, None), "haproxy_backend_max_total_time_seconds{": ("octavia_pool_max_total_time_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_requests_denied_total ": ("octavia_pool_requests_denied_total ", None, None), "haproxy_backend_requests_denied_total{": ("octavia_pool_requests_denied_total{", None, {"proxy=": "pool="}), "haproxy_backend_responses_denied_total ": ("octavia_pool_responses_denied_total ", None, None), "haproxy_backend_responses_denied_total{": ("octavia_pool_responses_denied_total{", None, {"proxy=": "pool="}), "haproxy_backend_connection_errors_total ": ("octavia_pool_connection_errors_total ", None, None), "haproxy_backend_connection_errors_total{": ("octavia_pool_connection_errors_total{", None, {"proxy=": "pool="}), "haproxy_backend_response_errors_total ": ("octavia_pool_response_errors_total ", None, None), "haproxy_backend_response_errors_total{": ("octavia_pool_response_errors_total{", None, {"proxy=": "pool="}), "haproxy_backend_retry_warnings_total ": ("octavia_pool_retry_warnings_total ", None, None), "haproxy_backend_retry_warnings_total{": ("octavia_pool_retry_warnings_total{", None, {"proxy=": "pool="}), "haproxy_backend_redispatch_warnings_total ": ("octavia_pool_redispatch_warnings_total ", None, None), "haproxy_backend_redispatch_warnings_total{": ("octavia_pool_redispatch_warnings_total{", None, {"proxy=": "pool="}), "haproxy_backend_failed_header_rewriting_total ": ("octavia_pool_failed_header_rewriting_total ", None, None), "haproxy_backend_failed_header_rewriting_total{": ("octavia_pool_failed_header_rewriting_total{", None, {"proxy=": "pool="}), "haproxy_backend_client_aborts_total ": ("octavia_pool_client_aborts_total ", None, None), "haproxy_backend_client_aborts_total{": ("octavia_pool_client_aborts_total{", None, {"proxy=": "pool="}), "haproxy_backend_server_aborts_total ": ("octavia_pool_member_aborts_total ", "# HELP octavia_pool_member_aborts_total Total number of data " "transfers aborted by the server.\n", None), "haproxy_backend_server_aborts_total{": ("octavia_pool_member_aborts_total{", None, {"proxy=": "pool="}), "haproxy_backend_active_servers ": ("octavia_pool_active_members ", "# HELP octavia_pool_active_members Current number of active " "members.\n", None), "haproxy_backend_active_servers{": ("octavia_pool_active_members{", None, {"proxy=": "pool="}), "haproxy_backend_backup_servers ": ("octavia_pool_backup_members ", "# HELP octavia_pool_backup_members Current number of backup " "members.\n", None), "haproxy_backend_backup_servers{": ("octavia_pool_backup_members{", None, {"proxy=": "pool="}), "haproxy_backend_check_up_down_total ": ("octavia_pool_check_up_down_total ", None, None), "haproxy_backend_check_up_down_total{": ("octavia_pool_check_up_down_total{", None, {"proxy=": "pool="}), "haproxy_backend_check_last_change_seconds ": ("octavia_pool_check_last_change_seconds ", None, None), "haproxy_backend_check_last_change_seconds{": ("octavia_pool_check_last_change_seconds{", None, {"proxy=": "pool="}), "haproxy_backend_downtime_seconds_total ": ("octavia_pool_downtime_seconds_total ", "# HELP octavia_pool_downtime_seconds_total Total downtime " "(in seconds) for the pool.\n", None), "haproxy_backend_downtime_seconds_total{": ("octavia_pool_downtime_seconds_total{", None, {"proxy=": "pool="}), "haproxy_backend_loadbalanced_total ": ("octavia_pool_loadbalanced_total ", "# HELP octavia_pool_loadbalanced_total Total number of times a " "pool was selected, either for new sessions, or when " "redispatching.\n", None), "haproxy_backend_loadbalanced_total{": ("octavia_pool_loadbalanced_total{", None, {"proxy=": "pool="}), "haproxy_backend_http_requests_total ": ("octavia_pool_http_requests_total ", None, None), "haproxy_backend_http_requests_total{": ("octavia_pool_http_requests_total{", None, {"proxy=": "pool="}), "haproxy_backend_http_responses_total ": ("octavia_pool_http_responses_total ", None, None), "haproxy_backend_http_responses_total{": ("octavia_pool_http_responses_total{", None, {"proxy=": "pool="}), "haproxy_backend_http_cache_lookups_total ": ("octavia_pool_http_cache_lookups_total ", None, None), "haproxy_backend_http_cache_lookups_total{": ("octavia_pool_http_cache_lookups_total{", None, {"proxy=": "pool="}), "haproxy_backend_http_cache_hits_total ": ("octavia_pool_http_cache_hits_total ", None, None), "haproxy_backend_http_cache_hits_total{": ("octavia_pool_http_cache_hits_total{", None, {"proxy=": "pool="}), "haproxy_backend_http_comp_bytes_in_total ": ("octavia_pool_http_comp_bytes_in_total ", None, None), "haproxy_backend_http_comp_bytes_in_total{": ("octavia_pool_http_comp_bytes_in_total{", None, {"proxy=": "pool="}), "haproxy_backend_http_comp_bytes_out_total ": ("octavia_pool_http_comp_bytes_out_total ", None, None), "haproxy_backend_http_comp_bytes_out_total{": ("octavia_pool_http_comp_bytes_out_total{", None, {"proxy=": "pool="}), "haproxy_backend_http_comp_bytes_bypassed_total ": ("octavia_pool_http_comp_bytes_bypassed_total ", None, None), "haproxy_backend_http_comp_bytes_bypassed_total{": ("octavia_pool_http_comp_bytes_bypassed_total{", None, {"proxy=": "pool="}), "haproxy_backend_http_comp_responses_total ": ("octavia_pool_http_comp_responses_total ", None, None), "haproxy_backend_http_comp_responses_total{": ("octavia_pool_http_comp_responses_total{", None, {"proxy=": "pool="}), # Member Metrics "haproxy_server_status ": ("octavia_member_status ", "# HELP octavia_member_status Current status of the member.\n", None), "haproxy_server_status{": ("octavia_member_status{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_current_sessions ": ("octavia_member_current_sessions ", None, None), "haproxy_server_current_sessions{": ("octavia_member_current_sessions{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_max_sessions ": ("octavia_member_max_sessions ", None, None), "haproxy_server_max_sessions{": ("octavia_member_max_sessions{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_limit_sessions ": ("octavia_member_limit_sessions ", None, None), "haproxy_server_limit_sessions{": ("octavia_member_limit_sessions{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_sessions_total ": ("octavia_member_sessions_total ", None, None), "haproxy_server_sessions_total{": ("octavia_member_sessions_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_max_session_rate ": ("octavia_member_max_session_rate ", None, None), "haproxy_server_max_session_rate{": ("octavia_member_max_session_rate{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_last_session_seconds ": ("octavia_member_last_session_seconds ", "# HELP octavia_member_last_session_seconds Number of seconds since " "last session assigned to the member.\n", None), "haproxy_server_last_session_seconds{": ("octavia_member_last_session_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_current_queue ": ("octavia_member_current_queue ", None, None), "haproxy_server_current_queue{": ("octavia_member_current_queue{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_max_queue ": ("octavia_member_max_queue ", None, None), "haproxy_server_max_queue{": ("octavia_member_max_queue{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_queue_limit ": ("octavia_member_queue_limit ", "# HELP octavia_member_queue_limit Configured maxqueue for the " "member (0 meaning no limit).\n", None), "haproxy_server_queue_limit{": ("octavia_member_queue_limit{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_bytes_in_total ": ("octavia_member_bytes_in_total ", None, None), "haproxy_server_bytes_in_total{": ("octavia_member_bytes_in_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_bytes_out_total ": ("octavia_member_bytes_out_total ", None, None), "haproxy_server_bytes_out_total{": ("octavia_member_bytes_out_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_queue_time_average_seconds ": ("octavia_member_queue_time_average_seconds ", None, None), "haproxy_server_queue_time_average_seconds{": ("octavia_member_queue_time_average_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_connect_time_average_seconds ": ("octavia_member_connect_time_average_seconds ", None, None), "haproxy_server_connect_time_average_seconds{": ("octavia_member_connect_time_average_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_response_time_average_seconds ": ("octavia_member_response_time_average_seconds ", None, None), "haproxy_server_response_time_average_seconds{": ("octavia_member_response_time_average_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_total_time_average_seconds ": ("octavia_member_total_time_average_seconds ", None, None), "haproxy_server_total_time_average_seconds{": ("octavia_member_total_time_average_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_max_queue_time_seconds ": ("octavia_member_max_queue_time_seconds ", None, None), "haproxy_server_max_queue_time_seconds{": ("octavia_member_max_queue_time_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_max_connect_time_seconds ": ("octavia_member_max_connect_time_seconds ", None, None), "haproxy_server_max_connect_time_seconds{": ("octavia_member_max_connect_time_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_max_response_time_seconds ": ("octavia_member_max_response_time_seconds ", "# HELP octavia_member_max_response_time_seconds Maximum observed " "time spent waiting for a member response.\n", None), "haproxy_server_max_response_time_seconds{": ("octavia_member_max_response_time_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_max_total_time_seconds ": ("octavia_member_max_total_time_seconds ", None, None), "haproxy_server_max_total_time_seconds{": ("octavia_member_max_total_time_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_connection_attempts_total ": ("octavia_member_connection_attempts_total ", None, None), "haproxy_server_connection_attempts_total{": ("octavia_member_connection_attempts_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_connection_reuses_total ": ("octavia_member_connection_reuses_total ", None, None), "haproxy_server_connection_reuses_total{": ("octavia_member_connection_reuses_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_responses_denied_total ": ("octavia_member_responses_denied_total ", None, None), "haproxy_server_responses_denied_total{": ("octavia_member_responses_denied_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_connection_errors_total ": ("octavia_member_connection_errors_total ", None, None), "haproxy_server_connection_errors_total{": ("octavia_member_connection_errors_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_response_errors_total ": ("octavia_member_response_errors_total ", None, None), "haproxy_server_response_errors_total{": ("octavia_member_response_errors_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_retry_warnings_total ": ("octavia_member_retry_warnings_total ", None, None), "haproxy_server_retry_warnings_total{": ("octavia_member_retry_warnings_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_redispatch_warnings_total ": ("octavia_member_redispatch_warnings_total ", None, None), "haproxy_server_redispatch_warnings_total{": ("octavia_member_redispatch_warnings_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_failed_header_rewriting_total ": ("octavia_member_failed_header_rewriting_total ", None, None), "haproxy_server_failed_header_rewriting_total{": ("octavia_member_failed_header_rewriting_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_client_aborts_total ": ("octavia_member_client_aborts_total ", None, None), "haproxy_server_client_aborts_total{": ("octavia_member_client_aborts_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_server_aborts_total ": ("octavia_member_server_aborts_total ", None, None), "haproxy_server_server_aborts_total{": ("octavia_member_server_aborts_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_weight ": ("octavia_member_weight ", "# HELP octavia_member_weight Member weight.\n", None), "haproxy_server_weight{": ("octavia_member_weight{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_check_failures_total ": ("octavia_member_check_failures_total ", "# HELP octavia_member_check_failures_total Total number of failed " "check (Only counts checks failed when the member is up).\n", None), "haproxy_server_check_failures_total{": ("octavia_member_check_failures_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_check_up_down_total ": ("octavia_member_check_up_down_total ", None, None), "haproxy_server_check_up_down_total{": ("octavia_member_check_up_down_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_downtime_seconds_total ": ("octavia_member_downtime_seconds_total ", "# HELP octavia_member_downtime_seconds_total Total downtime (in " "seconds) for the member.\n", None), "haproxy_server_downtime_seconds_total{": ("octavia_member_downtime_seconds_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_check_last_change_seconds ": ("octavia_member_check_last_change_seconds ", None, None), "haproxy_server_check_last_change_seconds{": ("octavia_member_check_last_change_seconds{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_current_throttle ": ("octavia_member_current_throttle ", "# HELP octavia_member_current_throttle Current throttle percentage " "for the member, when slowstart is active, or no value if not in " "slowstart.\n", None), "haproxy_server_current_throttle{": ("octavia_member_current_throttle{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_loadbalanced_total ": ("octavia_member_loadbalanced_total ", "# HELP octavia_member_loadbalanced_total Total number of times a " "member was selected, either for new sessions, or when " "redispatching.\n", None), "haproxy_server_loadbalanced_total{": ("octavia_member_loadbalanced_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_http_responses_total ": ("octavia_member_http_responses_total ", None, None), "haproxy_server_http_responses_total{": ("octavia_member_http_responses_total{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_idle_connections_current ": ("octavia_member_idle_connections_current ", None, None), "haproxy_server_idle_connections_current{": ("octavia_member_idle_connections_current{", None, {"proxy=": "pool=", "server=": "member="}), "haproxy_server_idle_connections_limit ": ("octavia_member_idle_connections_limit ", None, None), "haproxy_server_idle_connections_limit{": ("octavia_member_idle_connections_limit{", None, {"proxy=": "pool=", "server=": "member="}), } METRIC_KEYS = METRIC_MAP.keys() class PrometheusProxy(SimpleHTTPRequestHandler): protocol_version = 'HTTP/1.1' # No need to log every request through the proxy def log_request(self, *args, **kwargs): pass def _add_cpu_utilization(self, metrics_buffer): cpu_pcnt = (psutil.getloadavg()[0] / os.cpu_count()) * 100 metrics_buffer += ("# HELP octavia_loadbalancer_cpu Load balancer " "CPU utilization (percentage).\n") metrics_buffer += "# TYPE octavia_loadbalancer_cpu gauge\n" cpu_metric_string = f"octavia_loadbalancer_cpu {cpu_pcnt:.1f}\n" metrics_buffer += cpu_metric_string return metrics_buffer def _add_memory_utilization(self, metrics_buffer): mem_pcnt = psutil.virtual_memory()[2] metrics_buffer += ("# HELP octavia_loadbalancer_memory Load balancer " "memory utilization (percentage).\n") metrics_buffer += "# TYPE octavia_loadbalancer_memory gauge\n" mem_metric_string = f"octavia_loadbalancer_memory {mem_pcnt:.1f}\n" metrics_buffer += mem_metric_string return metrics_buffer def do_GET(self): metrics_buffer = "" metrics_buffer = self._add_cpu_utilization(metrics_buffer) metrics_buffer = self._add_memory_utilization(metrics_buffer) try: with urllib.request.urlopen(METRICS_URL) as source: # nosec lines = source.readlines() for line in lines: line = line.decode("utf-8") # Don't report metrics for the internal prometheus # proxy loop. The user facing listener will still be # reported. if "prometheus-exporter" in line: continue match = next((x for x in METRIC_KEYS if x in line), False) if match: map_tuple = METRIC_MAP[match] if map_tuple[1] and "HELP" in line: metrics_buffer += map_tuple[1] else: if map_tuple[2] and not line.startswith("#"): for key in map_tuple[2].keys(): line = line.replace(key, map_tuple[2][key]) metrics_buffer += line.replace(match, map_tuple[0]) elif PRINT_REJECTED: print(f"REJECTED: {line}") except Exception as e: print(str(e), flush=True) traceback.print_tb(e.__traceback__) self.send_response(502) self.send_header("connection", "close") self.end_headers() return self.send_response(200) self.send_header("cache-control", "no-cache") self.send_header("content-type", "text/plain; version=0.0.4") self.send_header("connection", "close") self.end_headers() self.wfile.write(metrics_buffer.encode("utf-8")) class SignalHandler: def __init__(self): signal.signal(signal.SIGINT, self.shutdown) signal.signal(signal.SIGTERM, self.shutdown) def shutdown(self, *args): EXIT_EVENT.set() def shutdown_thread(http): EXIT_EVENT.wait() http.shutdown() def main(): global PRINT_REJECTED try: if sys.argv[1] == "--rejected": PRINT_REJECTED = True except Exception: pass SignalHandler() while not EXIT_EVENT.is_set(): # The amphora-haproxy network namespace may not be present, so handle # it gracefully. try: with network_namespace.NetworkNamespace(consts.AMPHORA_NAMESPACE): httpd = ThreadingHTTPServer(('127.0.0.1', 9102), PrometheusProxy) shutdownthread = threading.Thread(target=shutdown_thread, args=(httpd,)) shutdownthread.start() httpd.daemon_threads = True print("Now serving on port 9102") httpd.serve_forever() except Exception: time.sleep(1)