From 1f46efc323dd102811d808f9e788c73a95e4f416 Mon Sep 17 00:00:00 2001 From: Jaganathan Palanisamy Date: Wed, 5 Jul 2017 09:40:06 -0400 Subject: [PATCH] CPU threads range format for derive parameters This change is to update CPU threads in numbers format to range format. Change-Id: I5421395b07421e1f6e47370ec76882f4133ff95f (cherry picked from commit cb3a932a488e6a6d1a3421ed6eec2030d0d78867) --- setup.cfg | 2 + tripleo_common/actions/derive_params.py | 121 ++++++++++++++++++ .../tests/actions/test_derive_params.py | 114 +++++++++++++++++ workbooks/derive_params_formulas.yaml | 108 +++++++++++++++- 4 files changed, 339 insertions(+), 6 deletions(-) diff --git a/setup.cfg b/setup.cfg index 79c48993a..f9c698625 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,6 +76,8 @@ mistral.actions = tripleo.deployment.config = tripleo_common.actions.deployment:OrchestrationDeployAction tripleo.deployment.deploy = tripleo_common.actions.deployment:DeployStackAction tripleo.deployment.overcloudrc = tripleo_common.actions.deployment:OvercloudRcAction + tripleo.derive_params.convert_number_to_range_list = tripleo_common.actions.derive_params:ConvertNumberToRangeListAction + tripleo.derive_params.convert_range_to_number_list = tripleo_common.actions.derive_params:ConvertRangeToNumberListAction tripleo.derive_params.get_dpdk_nics_numa_info = tripleo_common.actions.derive_params:GetDpdkNicsNumaInfoAction tripleo.derive_params.get_dpdk_core_list = tripleo_common.actions.derive_params:GetDpdkCoreListAction tripleo.derive_params.get_dpdk_socket_memory = tripleo_common.actions.derive_params:GetDpdkSocketMemoryAction diff --git a/tripleo_common/actions/derive_params.py b/tripleo_common/actions/derive_params.py index f1542dfa1..71541fb94 100644 --- a/tripleo_common/actions/derive_params.py +++ b/tripleo_common/actions/derive_params.py @@ -12,11 +12,16 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + +import logging import re from mistral_lib import actions from tripleo_common.actions import base +from tripleo_common import exception + +LOG = logging.getLogger(__name__) class GetDpdkNicsNumaInfoAction(base.TripleOAction): @@ -327,3 +332,119 @@ class GetDpdkSocketMemoryAction(base.TripleOAction): dpdk_socket_memory_list.append(socket_mem) return ','.join([str(sm) for sm in dpdk_socket_memory_list]) + + +class ConvertNumberToRangeListAction(base.TripleOAction): + """Converts number list into range list + + :param num_list: comma delimited number list as string + + :return: comma delimited range list as string + """ + + def __init__(self, num_list): + super(ConvertNumberToRangeListAction, self).__init__() + self.num_list = num_list + + # converts number list into range list. + # here input parameter and return value as list + # example: [12, 13, 14, 17] into ["12-14", "17"] + def convert_number_to_range_list(self, num_list): + num_list.sort() + range_list = [] + range_min = num_list[0] + for num in num_list: + next_val = num + 1 + if next_val not in num_list: + if range_min != num: + range_list.append(str(range_min) + '-' + str(num)) + else: + range_list.append(str(range_min)) + next_index = num_list.index(num) + 1 + if next_index < len(num_list): + range_min = num_list[next_index] + + # here, range_list is a list of strings + return range_list + + def run(self, context): + try: + if not self.num_list: + err_msg = ("Input param 'num_list' is blank.") + raise exception.DeriveParamsError(err_msg) + + try: + # splitting a string (comma delimited list) into + # list of numbers + # example: "12,13,14,17" string into [12,13,14,17] + num_list = [int(num.strip(' ')) + for num in self.num_list.split(",")] + except ValueError as exc: + err_msg = ("Invalid number in input param " + "'num_list': %s" % exc) + raise exception.DeriveParamsError(err_msg) + + range_list = self.convert_number_to_range_list(num_list) + except exception.DeriveParamsError as err: + LOG.error('Derive Params Error: %s', err) + return actions.Result(error=str(err)) + + # converts into comma delimited range list as string + return ','.join(range_list) + + +class ConvertRangeToNumberListAction(base.TripleOAction): + """Converts range list to integer list + + :param range_list: comma delimited range list as string / list + + :return: comma delimited number list as string + """ + + def __init__(self, range_list): + super(ConvertRangeToNumberListAction, self).__init__() + self.range_list = range_list + + # converts range list into number list + # here input parameter and return value as list + # example: ["12-14", "^13", "17"] into [12, 14, 17] + def convert_range_to_number_list(self, range_list): + num_list = [] + exclude_num_list = [] + try: + for val in range_list: + val = val.strip(' ') + if '^' in val: + exclude_num_list.append(int(val[1:])) + elif '-' in val: + split_list = val.split("-") + range_min = int(split_list[0]) + range_max = int(split_list[1]) + num_list.extend(range(range_min, (range_max + 1))) + else: + num_list.append(int(val)) + except ValueError as exc: + err_msg = ("Invalid number in input param " + "'range_list': %s" % exc) + raise exception.DeriveParamsError(err_msg) + + # here, num_list is a list of integers + return [num for num in num_list if num not in exclude_num_list] + + def run(self, context): + try: + if not self.range_list: + err_msg = ("Input param 'range_list' is blank.") + raise exception.DeriveParamsError(err_msg) + range_list = self.range_list + # converts into python list if range_list is not list type + if not isinstance(range_list, list): + range_list = self.range_list.split(",") + + num_list = self.convert_range_to_number_list(range_list) + except exception.DeriveParamsError as err: + LOG.error('Derive Params Error: %s', err) + return actions.Result(error=str(err)) + + # converts into comma delimited number list as string + return ','.join([str(num) for num in num_list]) diff --git a/tripleo_common/tests/actions/test_derive_params.py b/tripleo_common/tests/actions/test_derive_params.py index 02057ae55..d57dda267 100644 --- a/tripleo_common/tests/actions/test_derive_params.py +++ b/tripleo_common/tests/actions/test_derive_params.py @@ -422,3 +422,117 @@ class GetDpdkSocketMemoryActionTest(base.TestCase): dpdk_nics_numa_info, numa_nodes, overhead, packet_size_in_buffer) result = action.run(mock_ctx) self.assertEqual(result, expected_result) + + +class ConvertNumberToRangeListActionTest(base.TestCase): + + def test_run_with_ranges(self): + num_list = "0,22,23,24,25,60,65,66,67" + expected_result = "0,22-25,60,65-67" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertNumberToRangeListAction(num_list) + result = action.run(mock_ctx) + self.assertEqual(result, expected_result) + + @mock.patch.object(actions, 'Result', autospec=True) + def test_run_with_no_range(self, mock_actions): + num_list = "0,22,24,60,65,67" + expected_result = "0,22,24,60,65,67" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertNumberToRangeListAction(num_list) + result = action.run(mock_ctx) + self.assertEqual(result, expected_result) + + @mock.patch.object(actions, 'Result', autospec=True) + def test_run_with_empty_input(self, mock_actions): + num_list = "" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertNumberToRangeListAction(num_list) + action.run(mock_ctx) + msg = "Input param 'num_list' is blank." + mock_actions.assert_called_once_with(error=msg) + + @mock.patch.object(actions, 'Result', autospec=True) + def test_run_with_invalid_input(self, mock_actions): + num_list = ",d" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertNumberToRangeListAction(num_list) + action.run(mock_ctx) + msg = ("Invalid number in input param 'num_list': invalid " + "literal for int() with base 10: ''") + mock_actions.assert_called_once_with(error=msg) + + +class ConvertRangeToNumberListActionTest(base.TestCase): + + def test_run_with_ranges_in_comma_delimited_str(self): + range_list = "24-27,60,65-67" + expected_result = "24,25,26,27,60,65,66,67" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertRangeToNumberListAction(range_list) + result = action.run(mock_ctx) + self.assertEqual(result, expected_result) + + def test_run_with_ranges_in_comma_delimited_list(self): + range_list = ['24-27', '60', '65-67'] + expected_result = "24,25,26,27,60,65,66,67" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertRangeToNumberListAction(range_list) + result = action.run(mock_ctx) + self.assertEqual(result, expected_result) + + @mock.patch.object(actions, 'Result', autospec=True) + def test_run_with_ranges_exclude_num(self, mock_actions): + range_list = "24-27,^25,60,65-67" + expected_result = "24,26,27,60,65,66,67" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertRangeToNumberListAction(range_list) + result = action.run(mock_ctx) + self.assertEqual(result, expected_result) + + def test_run_with_no_ranges(self): + range_list = "24,25,26,27,60,65,66,67" + expected_result = "24,25,26,27,60,65,66,67" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertRangeToNumberListAction(range_list) + result = action.run(mock_ctx) + self.assertEqual(result, expected_result) + + @mock.patch.object(actions, 'Result', autospec=True) + def test_run_with_empty_input(self, mock_actions): + range_list = "" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertRangeToNumberListAction(range_list) + action.run(mock_ctx) + msg = "Input param 'range_list' is blank." + mock_actions.assert_called_once_with(error=msg) + + @mock.patch.object(actions, 'Result', autospec=True) + def test_run_with_invalid_input(self, mock_actions): + range_list = ",d" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertRangeToNumberListAction(range_list) + action.run(mock_ctx) + msg = ("Invalid number in input param 'range_list': invalid " + "literal for int() with base 10: ''") + mock_actions.assert_called_once_with(error=msg) + + @mock.patch.object(actions, 'Result', autospec=True) + def test_run_with_invalid_exclude_number(self, mock_actions): + range_list = "12-15,^17" + expected_result = "12,13,14,15" + + mock_ctx = mock.MagicMock() + action = derive_params.ConvertRangeToNumberListAction(range_list) + result = action.run(mock_ctx) + self.assertEqual(result, expected_result) diff --git a/workbooks/derive_params_formulas.yaml b/workbooks/derive_params_formulas.yaml index 36a691d16..1e00f29b5 100644 --- a/workbooks/derive_params_formulas.yaml +++ b/workbooks/derive_params_formulas.yaml @@ -73,19 +73,37 @@ workflows: publish: pmd_cpus: <% task().result %> on-success: - - get_host_cpus: <% $.pmd_cpus %> + - get_pmd_cpus_range_list: <% $.pmd_cpus %> - set_status_failed_get_pmd_cpus: <% not $.pmd_cpus %> on-error: set_status_failed_on_error_get_pmd_cpus + get_pmd_cpus_range_list: + action: tripleo.derive_params.convert_number_to_range_list + input: + num_list: <% $.pmd_cpus %> + publish: + pmd_cpus: <% task().result %> + on-success: get_host_cpus + on-error: set_status_failed_get_pmd_cpus_range_list + get_host_cpus: action: tripleo.derive_params.get_host_cpus_list inspect_data=<% $.hw_data %> publish: host_cpus: <% task().result %> on-success: - - get_sock_mem: <% $.host_cpus %> + - get_host_cpus_range_list: <% $.host_cpus %> - set_status_failed_get_host_cpus: <% not $.host_cpus %> on-error: set_status_failed_on_error_get_host_cpus + get_host_cpus_range_list: + action: tripleo.derive_params.convert_number_to_range_list + input: + num_list: <% $.host_cpus %> + publish: + host_cpus: <% task().result %> + on-success: get_sock_mem + on-error: set_status_failed_get_host_cpus_range_list + get_sock_mem: action: tripleo.derive_params.get_dpdk_socket_memory input: @@ -176,6 +194,12 @@ workflows: message: <% task(get_pmd_cpus).result %> on-success: fail + set_status_failed_get_pmd_cpus_range_list: + publish: + status: FAILED + message: <% task(get_pmd_cpus_range_list).result %> + on-success: fail + set_status_failed_get_host_cpus: publish: status: FAILED @@ -188,6 +212,12 @@ workflows: message: <% task(get_host_cpus).result %> on-success: fail + set_status_failed_get_host_cpus_range_list: + publish: + status: FAILED + message: <% task(get_host_cpus_range_list).result %> + on-success: fail + set_status_failed_get_sock_mem: publish: status: FAILED @@ -243,22 +273,64 @@ workflows: get_host_dpdk_combined_cpus: publish: - host_dpdk_combined_cpus: <% let(params => $.role_derive_params) -> concat($params.get('OvsPmdCoreList', ''), ',', $params.get('OvsDpdkCoreList', '')).split(",").select(int($)) %> + host_dpdk_combined_cpus: <% concat($.role_derive_params.get('OvsPmdCoreList', ''), ',', $.role_derive_params.get('OvsDpdkCoreList', '')) %> on-success: - - get_nova_cpus: <% $.host_dpdk_combined_cpus %> + - get_host_dpdk_combined_cpus_num_list: <% $.host_dpdk_combined_cpus %> - set_status_failed_get_host_dpdk_combined_cpus: <% not $.host_dpdk_combined_cpus %> + get_host_dpdk_combined_cpus_num_list: + action: tripleo.derive_params.convert_range_to_number_list + input: + range_list: <% $.host_dpdk_combined_cpus %> + publish: + host_dpdk_combined_cpus: <% task().result %> + on-success: get_nova_cpus + on-error: set_status_failed_get_host_dpdk_combined_cpus_num_list + get_nova_cpus: publish: - nova_cpus: <% let(invalid_threads => $.host_dpdk_combined_cpus) -> $.cpus.select($.thread_siblings).flatten().where(not $ in $invalid_threads).join(',') %> + nova_cpus: <% let(reserved_cpus => $.host_dpdk_combined_cpus.split(',')) -> $.cpus.select($.thread_siblings).flatten().where(not (str($) in $reserved_cpus)).join(',') %> on-success: - get_isol_cpus: <% $.nova_cpus %> - set_status_failed_get_nova_cpus: <% not $.nova_cpus %> + # concatinates OvsPmdCoreList range format and NovaVcpuPinSet in range format. it may not be in perfect range format. + # example: concatinates '12-15,19' and 16-18' ranges '12-15,19,16-18' get_isol_cpus: publish: - isol_cpus: <% let(params => $.role_derive_params) -> concat($params.get('OvsPmdCoreList',''), ',', $.nova_cpus) %> + isol_cpus: <% concat($.role_derive_params.get('OvsPmdCoreList',''), ',', $.nova_cpus) %> + on-success: get_isol_cpus_num_list + + # Gets the isol_cpus in the number list + # example: '12-15,19,16-18' into '12,13,14,15,16,17,18,19' + get_isol_cpus_num_list: + action: tripleo.derive_params.convert_range_to_number_list + input: + range_list: <% $.isol_cpus %> + publish: + isol_cpus: <% task().result %> + on-success: get_nova_cpus_range_list + on-error: set_status_failed_get_isol_cpus_num_list + + get_nova_cpus_range_list: + action: tripleo.derive_params.convert_number_to_range_list + input: + num_list: <% $.nova_cpus %> + publish: + nova_cpus: <% task().result %> + on-success: get_isol_cpus_range_list + on-error: set_status_failed_get_nova_cpus_range_list + + # converts number format isol_cpus into range format + # example: '12,13,14,15,16,17,18,19' into '12-19' + get_isol_cpus_range_list: + action: tripleo.derive_params.convert_number_to_range_list + input: + num_list: <% $.isol_cpus %> + publish: + isol_cpus: <% task().result %> on-success: get_host_mem + on-error: set_status_failed_get_isol_cpus_range_list get_host_mem: publish: @@ -317,12 +389,36 @@ workflows: message: 'Unable to combine host and dpdk cpus list' on-success: fail + set_status_failed_get_host_dpdk_combined_cpus_num_list: + publish: + status: FAILED + message: <% task(get_host_dpdk_combined_cpus_num_list).result %> + on-success: fail + set_status_failed_get_nova_cpus: publish: status: FAILED message: 'Unable to determine nova vcpu pin set' on-success: fail + set_status_failed_get_nova_cpus_range_list: + publish: + status: FAILED + message: <% task(get_nova_cpus_range_list).result %> + on-success: fail + + set_status_failed_get_isol_cpus_num_list: + publish: + status: FAILED + message: <% task(get_isol_cpus_num_list).result %> + on-success: fail + + set_status_failed_get_isol_cpus_range_list: + publish: + status: FAILED + message: <% task(get_isol_cpus_range_list).result %> + on-success: fail + set_status_failed_check_default_hugepage_supported: publish: status: FAILED