diff --git a/whitebox_neutron_tempest_plugin/tests/scenario/test_security_group_logging.py b/whitebox_neutron_tempest_plugin/tests/scenario/test_security_group_logging.py index 99a5d869..b2e5f928 100644 --- a/whitebox_neutron_tempest_plugin/tests/scenario/test_security_group_logging.py +++ b/whitebox_neutron_tempest_plugin/tests/scenario/test_security_group_logging.py @@ -13,8 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. +import math from pprint import pformat import random +import re import socket import time @@ -621,41 +623,91 @@ class BaseSecGroupLoggingTest( fetch_limit_cmd.format('burst'))) rate_limit = int(self.master_cont_cmd_executor( fetch_limit_cmd.format('rate'))) - # 14) send excessive ICMP requests to VM for 1 second - self.start_track_log(vm_a['hv_ssh_client']) - self.validate_command( - 'sudo ping {} -i 0.002 -c 500 | tail -n4'.format(vm_a['fip']), - pattern=r' 0% packet loss', - local_shell=True) - self.retrieve_tracked_log(vm_a['hv_ssh_client']) + # 14) send excessive ICMP requests to VM for 1 second (testing burst) + # NOTE(mblue): congestion on specific setups may delay few seconds + # instead of 1s, optional few attempts done to get closer times to 1s + expected_offset = 0.1 + expected_ping_duration = 1 + attempts_1s = 5 + for attempt in range(1, attempts_1s + 1): + self.start_track_log(vm_a['hv_ssh_client']) + ping_report = self.validate_command( + 'sudo ping {} -i 0.002 -c 500 | tail -n4'.format(vm_a['fip']), + pattern=r' 0% packet loss', + local_shell=True) + # due to lots of requests and responses wait time, + # difference beyond 1 second is possible, should be accounted for + try: + ping_ms = int( + re.search( + r'packet loss, time (\d+)ms', ping_report).group(1)) + except (TypeError, AttributeError): + LOG.exception( + 'ms value failed to extract from ping output:\n' + '```\n%s\n```', + ping_report) + ping_extra = max(0, ping_ms / 1000 - expected_ping_duration) + self.retrieve_tracked_log(vm_a['hv_ssh_client']) + # try reaching only 10% more time according to error offset tested + if ping_extra < expected_offset: + break + if attempt == attempts_1s: + LOG.warning( + 'Logging over 1s delayed %d times for many seconds ' + '(latest ping extra time is %.3f)', + attempts_1s, + ping_extra) # 15) verify log entries amount equals to: - # rate limit + burst limit (up to 10% offset allowed) + # rate limit + burst limit (up to 10% expected offset allowed) limits_fail_msg = "rate or burst log limits not enforced correctly" - quick_test_target = burst_limit + rate_limit + quick_test_target = \ + rate_limit * (expected_ping_duration + ping_extra) + burst_limit quick_test_logs_amount = self._hypervisors_counts[ vm_a['hv_ssh_client'].host]['test_logs_amount'] - offset = 0.1 + # in very congested setups, last resort is allowing higher error + # offset up to a sensible limit of 33% and only in case it is needed + # (additionally to prior multiple attempts for short ping delay, + # and adjusting limits to extra ping time in hindsight) + final_offset = min( + 1 / 3, + expected_offset if ping_extra < expected_offset else ping_extra) + LOG.debug('final offset is %.3f', final_offset) self.assertIn(quick_test_logs_amount, - range(round((1 - offset) * quick_test_target), - round((1 + offset) * quick_test_target)), + range(math.floor((1 - final_offset) * quick_test_target), + math.ceil((1 + final_offset) * quick_test_target)), limits_fail_msg) - # 16) send excessive ICMP requests to VM for 60 seconds + # 16) send excessive ICMP requests to VM for 60 seconds (testing rate) + expected_ping_duration = 60 self.start_track_log(vm_a['hv_ssh_client']) - self.validate_command( + ping_report = self.validate_command( 'sudo ping {} -i 0.005 -c 12000 | tail -n4'.format(vm_a['fip']), pattern=r' 0% packet loss', - timeout=70, + timeout=90, local_shell=True) + # NOTE(mblue): due to lots of requests and their responses wait time, + # difference beyond 60 seconds is possible, should be accounted for. + try: + ping_ms = int( + re.search( + r'packet loss, time (\d+)ms', ping_report).group(1)) + except (TypeError, AttributeError): + LOG.exception( + 'ms value failed to extract from ping output:\n' + '```\n%s\n```', + ping_report) + ping_extra = max(0, ping_ms / 1000 - expected_ping_duration) self.retrieve_tracked_log(vm_a['hv_ssh_client']) # 17) verify log entries amount equals to: - # rate limit * 60 + burst limit (up to 5% offset allowed) - long_test_target = rate_limit * 60 + burst_limit + # rate limit * (60 + optional extra ping time) + burst limit + # (up to 5% final offset allowed). + long_test_target = \ + rate_limit * (expected_ping_duration + ping_extra) + burst_limit long_test_logs_amount = self._hypervisors_counts[ vm_a['hv_ssh_client'].host]['test_logs_amount'] - offset = 0.05 + final_offset = 0.05 self.assertIn(long_test_logs_amount, - range(round((1 - offset) * long_test_target), - round((1 + offset) * long_test_target)), + range(math.floor((1 - final_offset) * long_test_target), + math.ceil((1 + final_offset) * long_test_target)), limits_fail_msg) self._verify_duplicate_entries(vm_a)