From 69f1753903c79fa97fdbaee3f42d78328f936020 Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Fri, 18 Jan 2019 17:07:36 -0800 Subject: [PATCH] Add compute flavor support to the amphora driver This patch adds a new flavor capability to the amphora driver called 'compute_flavor'. This allows an amphora flavor to specify a compute (nova) flavor to be used for the load balancer instances. Change-Id: I8626eebd906c935a47d3e3510d1dfefae307c4e9 --- octavia/api/drivers/amphora_driver/driver.py | 12 +++++++++ .../drivers/amphora_driver/flavor_schema.py | 4 +++ octavia/common/constants.py | 1 + octavia/compute/compute_base.py | 11 ++++++++ octavia/compute/drivers/noop_driver/driver.py | 8 ++++++ octavia/compute/drivers/nova_driver.py | 19 ++++++++++++++ .../controller/worker/controller_worker.py | 7 +++++ .../controller/worker/flows/amphora_flows.py | 9 +++++-- .../controller/worker/tasks/compute_tasks.py | 15 ++++++++--- .../unit/compute/drivers/test_nova_driver.py | 17 ++++++++++++ .../worker/flows/test_amphora_flows.py | 26 +++++++++---------- .../worker/flows/test_load_balancer_flows.py | 4 +-- .../worker/test_controller_worker.py | 12 +++++++-- ...te-flavor-capability-ab202697a7fbdc3d.yaml | 8 ++++++ 14 files changed, 130 insertions(+), 23 deletions(-) create mode 100644 releasenotes/notes/add-compute-flavor-capability-ab202697a7fbdc3d.yaml diff --git a/octavia/api/drivers/amphora_driver/driver.py b/octavia/api/drivers/amphora_driver/driver.py index 875fb8d63f..39d0676130 100644 --- a/octavia/api/drivers/amphora_driver/driver.py +++ b/octavia/api/drivers/amphora_driver/driver.py @@ -18,6 +18,7 @@ from jsonschema import validate from oslo_config import cfg from oslo_log import log as logging import oslo_messaging as messaging +from stevedore import driver as stevedore_driver from octavia.api.drivers.amphora_driver import flavor_schema from octavia.api.drivers import data_models as driver_dm @@ -311,3 +312,14 @@ class AmphoraProviderDriver(driver_base.ProviderDriver): 'due to: {}'.format(str(e)), operator_fault_string='Failed to validate the flavor metadata ' 'due to: {}'.format(str(e))) + compute_flavor = flavor_dict.get(consts.COMPUTE_FLAVOR, None) + if compute_flavor: + compute_driver = stevedore_driver.DriverManager( + namespace='octavia.compute.drivers', + name=CONF.controller_worker.compute_driver, + invoke_on_load=True + ).driver + + # TODO(johnsom) Fix this to raise a NotFound error + # when the octavia-lib supports it. + compute_driver.validate_flavor(compute_flavor) diff --git a/octavia/api/drivers/amphora_driver/flavor_schema.py b/octavia/api/drivers/amphora_driver/flavor_schema.py index 9755bf6a3a..7231fc998a 100644 --- a/octavia/api/drivers/amphora_driver/flavor_schema.py +++ b/octavia/api/drivers/amphora_driver/flavor_schema.py @@ -39,6 +39,10 @@ SUPPORTED_FLAVOR_SCHEMA = { "SINGLE - One amphora per load balancer. " "ACTIVE_STANDBY - Two amphora per load balancer.", "enum": list(consts.SUPPORTED_LB_TOPOLOGIES) + }, + consts.COMPUTE_FLAVOR: { + "type": "string", + "description": "The compute driver flavor ID." } } } diff --git a/octavia/common/constants.py b/octavia/common/constants.py index 370ce69bd1..5c53c21410 100644 --- a/octavia/common/constants.py +++ b/octavia/common/constants.py @@ -568,3 +568,4 @@ FLAVOR_DATA = 'flavor_data' # Flavor metadata LOADBALANCER_TOPOLOGY = 'loadbalancer_topology' +COMPUTE_FLAVOR = 'compute_flavor' diff --git a/octavia/compute/compute_base.py b/octavia/compute/compute_base.py index 071eefd86c..3274db339c 100644 --- a/octavia/compute/compute_base.py +++ b/octavia/compute/compute_base.py @@ -121,3 +121,14 @@ class ComputeBase(object): :raises: Exception """ pass + + @abc.abstractmethod + def validate_flavor(self, flavor_id): + """Validates that a compute flavor exists. + + :param flavor_id: ID of the compute flavor. + :return: None + :raises: NotFound + :raises: NotImplementedError + """ + pass diff --git a/octavia/compute/drivers/noop_driver/driver.py b/octavia/compute/drivers/noop_driver/driver.py index de2a680b6d..6683529e0d 100644 --- a/octavia/compute/drivers/noop_driver/driver.py +++ b/octavia/compute/drivers/noop_driver/driver.py @@ -106,6 +106,11 @@ class NoopManager(object): self.computeconfig[(compute_id, port_id)] = ( compute_id, port_id, 'detach_port') + def validate_flavor(self, flavor_id): + LOG.debug("Compute %s no-op, validate_flavor flavor_id %s", + self.__class__.__name__, flavor_id) + self.computeconfig[flavor_id] = (flavor_id, 'validate_flavor') + class NoopComputeDriver(driver_base.ComputeBase): def __init__(self): @@ -147,3 +152,6 @@ class NoopComputeDriver(driver_base.ComputeBase): def detach_port(self, compute_id, port_id): self.driver.detach_port(compute_id, port_id) + + def validate_flavor(self, flavor_id): + self.driver.validate_flavor(flavor_id) diff --git a/octavia/compute/drivers/nova_driver.py b/octavia/compute/drivers/nova_driver.py index 9ddbad8244..58f497b6d9 100644 --- a/octavia/compute/drivers/nova_driver.py +++ b/octavia/compute/drivers/nova_driver.py @@ -88,6 +88,7 @@ class VirtualMachineManager(compute_base.ComputeBase): cacert=CONF.glance.ca_certificates_file) self.manager = self._nova_client.servers self.server_groups = self._nova_client.server_groups + self.flavor_manager = self._nova_client.flavors def build(self, name="amphora_name", amphora_flavor=None, image_id=None, image_tag=None, image_owner=None, @@ -327,3 +328,21 @@ class VirtualMachineManager(compute_base.ComputeBase): 'with compute ID {compute_id}. ' 'Skipping.'.format(port_id=port_id, compute_id=compute_id)) + + def validate_flavor(self, flavor_id): + """Validates that a flavor exists in nova. + + :param flavor_id: ID of the flavor to lookup in nova. + :raises: NotFound + :returns: None + """ + try: + self.flavor_manager.get(flavor_id) + except nova_exceptions.NotFound: + LOG.info('Flavor {} was not found in nova.'.format(flavor_id)) + raise exceptions.InvalidSubresource(resource='Nova flavor', + id=flavor_id) + except Exception as e: + LOG.exception('Nova reports a failure getting flavor details for ' + 'flavor ID {0}: {1}'.format(flavor_id, str(e))) + raise diff --git a/octavia/controller/worker/controller_worker.py b/octavia/controller/worker/controller_worker.py index 6e950597c0..5a306acb17 100644 --- a/octavia/controller/worker/controller_worker.py +++ b/octavia/controller/worker/controller_worker.py @@ -69,6 +69,7 @@ class ControllerWorker(base_taskflow.BaseTaskFlowEngine): self._pool_repo = repo.PoolRepository() self._l7policy_repo = repo.L7PolicyRepository() self._l7rule_repo = repo.L7RuleRepository() + self._flavor_repo = repo.FlavorRepository() self._exclude_result_logging_tasks = ( constants.ROLE_STANDALONE + '-' + @@ -840,6 +841,12 @@ class ControllerWorker(base_taskflow.BaseTaskFlowEngine): db_apis.get_session(), amp.id) if CONF.nova.enable_anti_affinity and lb: stored_params[constants.SERVER_GROUP_ID] = lb.server_group_id + if lb.flavor_id: + stored_params[constants.FLAVOR] = ( + self._flavor_repo.get_flavor_metadata_dict( + db_apis.get_session(), lb.flavor_id)) + else: + stored_params[constants.FLAVOR] = {} failover_amphora_tf = self._taskflow_load( self._amphora_flows.get_failover_flow( diff --git a/octavia/controller/worker/flows/amphora_flows.py b/octavia/controller/worker/flows/amphora_flows.py index 7bdaed30a1..bfe529f47c 100644 --- a/octavia/controller/worker/flows/amphora_flows.py +++ b/octavia/controller/worker/flows/amphora_flows.py @@ -55,11 +55,12 @@ class AmphoraFlows(object): create_amphora_flow.add(compute_tasks.CertComputeCreate( requires=(constants.AMPHORA_ID, constants.SERVER_PEM, - constants.BUILD_TYPE_PRIORITY), + constants.BUILD_TYPE_PRIORITY, constants.FLAVOR), provides=constants.COMPUTE_ID)) else: create_amphora_flow.add(compute_tasks.ComputeCreate( - requires=(constants.AMPHORA_ID, constants.BUILD_TYPE_PRIORITY), + requires=(constants.AMPHORA_ID, constants.BUILD_TYPE_PRIORITY, + constants.FLAVOR), provides=constants.COMPUTE_ID)) create_amphora_flow.add(database_tasks.MarkAmphoraBootingInDB( requires=(constants.AMPHORA_ID, constants.COMPUTE_ID))) @@ -140,6 +141,7 @@ class AmphoraFlows(object): constants.SERVER_PEM, constants.BUILD_TYPE_PRIORITY, constants.SERVER_GROUP_ID, + constants.FLAVOR ), provides=constants.COMPUTE_ID)) else: @@ -149,6 +151,7 @@ class AmphoraFlows(object): constants.AMPHORA_ID, constants.SERVER_PEM, constants.BUILD_TYPE_PRIORITY, + constants.FLAVOR ), provides=constants.COMPUTE_ID)) else: @@ -159,6 +162,7 @@ class AmphoraFlows(object): constants.AMPHORA_ID, constants.BUILD_TYPE_PRIORITY, constants.SERVER_GROUP_ID, + constants.FLAVOR ), provides=constants.COMPUTE_ID)) else: @@ -167,6 +171,7 @@ class AmphoraFlows(object): requires=( constants.AMPHORA_ID, constants.BUILD_TYPE_PRIORITY, + constants.FLAVOR ), provides=constants.COMPUTE_ID)) diff --git a/octavia/controller/worker/tasks/compute_tasks.py b/octavia/controller/worker/tasks/compute_tasks.py index 5b2342aa13..844b660827 100644 --- a/octavia/controller/worker/tasks/compute_tasks.py +++ b/octavia/controller/worker/tasks/compute_tasks.py @@ -49,7 +49,7 @@ class ComputeCreate(BaseComputeTask): def execute(self, amphora_id, config_drive_files=None, build_type_priority=constants.LB_CREATE_NORMAL_PRIORITY, - server_group_id=None, ports=None): + server_group_id=None, ports=None, flavor=None): """Create an amphora :returns: an amphora @@ -68,6 +68,13 @@ class ComputeCreate(BaseComputeTask): ssh_access = CONF.controller_worker.amp_ssh_access_allowed key_name = None if not ssh_access else key_name + # Apply an Octavia flavor customizations + if flavor: + amp_compute_flavor = flavor.get( + constants.COMPUTE_FLAVOR, CONF.controller_worker.amp_flavor_id) + else: + amp_compute_flavor = CONF.controller_worker.amp_flavor_id + try: if CONF.haproxy_amphora.build_rate_limit != -1: self.rate_limit.add_to_build_request_queue( @@ -84,7 +91,7 @@ class ComputeCreate(BaseComputeTask): compute_id = self.compute.build( name="amphora-" + amphora_id, - amphora_flavor=CONF.controller_worker.amp_flavor_id, + amphora_flavor=amp_compute_flavor, image_id=CONF.controller_worker.amp_image_id, image_tag=CONF.controller_worker.amp_image_tag, image_owner=CONF.controller_worker.amp_image_owner_id, @@ -125,7 +132,7 @@ class ComputeCreate(BaseComputeTask): class CertComputeCreate(ComputeCreate): def execute(self, amphora_id, server_pem, build_type_priority=constants.LB_CREATE_NORMAL_PRIORITY, - server_group_id=None, ports=None): + server_group_id=None, ports=None, flavor=None): """Create an amphora :returns: an amphora @@ -140,7 +147,7 @@ class CertComputeCreate(ComputeCreate): return super(CertComputeCreate, self).execute( amphora_id, config_drive_files=config_drive_files, build_type_priority=build_type_priority, - server_group_id=server_group_id, ports=ports) + server_group_id=server_group_id, ports=ports, flavor=flavor) class DeleteAmphoraeOnLoadBalancer(BaseComputeTask): diff --git a/octavia/tests/unit/compute/drivers/test_nova_driver.py b/octavia/tests/unit/compute/drivers/test_nova_driver.py index 1354ae60a9..0f331e6329 100644 --- a/octavia/tests/unit/compute/drivers/test_nova_driver.py +++ b/octavia/tests/unit/compute/drivers/test_nova_driver.py @@ -126,6 +126,8 @@ class TestNovaClient(base.TestCase): self.manager.manager = mock.MagicMock() self.manager.server_groups = mock.MagicMock() self.manager._nova_client = mock.MagicMock() + self.manager.flavor_manager = mock.MagicMock() + self.manager.flavor_manager.get = mock.MagicMock() self.nova_response.interface_list.side_effect = [[self.interface_list]] self.manager.manager.get.return_value = self.nova_response @@ -149,6 +151,7 @@ class TestNovaClient(base.TestCase): self.port_id = uuidutils.generate_uuid() self.compute_id = uuidutils.generate_uuid() self.network_id = uuidutils.generate_uuid() + self.flavor_id = uuidutils.generate_uuid() super(TestNovaClient, self).setUp() @@ -373,3 +376,17 @@ class TestNovaClient(base.TestCase): self.manager.manager.interface_detach.side_effect = [Exception] self.manager.detach_port(self.compute_id, self.port_id) + + def test_validate_flavor(self): + self.manager.validate_flavor(self.flavor_id) + self.manager.flavor_manager.get.assert_called_with(self.flavor_id) + + def test_validate_flavor_with_exception(self): + self.manager.flavor_manager.get.side_effect = [ + nova_exceptions.NotFound(404), exceptions.OctaviaException] + self.assertRaises(exceptions.InvalidSubresource, + self.manager.validate_flavor, + "bogus") + self.assertRaises(exceptions.OctaviaException, + self.manager.validate_flavor, + "bogus") diff --git a/octavia/tests/unit/controller/worker/flows/test_amphora_flows.py b/octavia/tests/unit/controller/worker/flows/test_amphora_flows.py index 57ce767d80..418598ebec 100644 --- a/octavia/tests/unit/controller/worker/flows/test_amphora_flows.py +++ b/octavia/tests/unit/controller/worker/flows/test_amphora_flows.py @@ -57,7 +57,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(1, len(amp_flow.requires)) + self.assertEqual(2, len(amp_flow.requires)) def test_get_create_amphora_flow_cert(self, mock_get_net_driver): self.AmpFlow = amphora_flows.AmphoraFlows() @@ -71,7 +71,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.COMPUTE_ID, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(1, len(amp_flow.requires)) + self.assertEqual(2, len(amp_flow.requires)) def test_get_create_amphora_for_lb_flow(self, mock_get_net_driver): @@ -89,7 +89,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(2, len(amp_flow.requires)) + self.assertEqual(3, len(amp_flow.requires)) def test_get_cert_create_amphora_for_lb_flow(self, mock_get_net_driver): @@ -109,7 +109,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(2, len(amp_flow.requires)) + self.assertEqual(3, len(amp_flow.requires)) def test_get_cert_master_create_amphora_for_lb_flow( self, mock_get_net_driver): @@ -130,7 +130,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(2, len(amp_flow.requires)) + self.assertEqual(3, len(amp_flow.requires)) def test_get_cert_master_rest_anti_affinity_create_amphora_for_lb_flow( self, mock_get_net_driver): @@ -149,7 +149,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(3, len(amp_flow.requires)) + self.assertEqual(4, len(amp_flow.requires)) self.conf.config(group="nova", enable_anti_affinity=False) def test_get_cert_backup_create_amphora_for_lb_flow( @@ -170,7 +170,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(2, len(amp_flow.requires)) + self.assertEqual(3, len(amp_flow.requires)) def test_get_cert_bogus_create_amphora_for_lb_flow( self, mock_get_net_driver): @@ -190,7 +190,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(2, len(amp_flow.requires)) + self.assertEqual(3, len(amp_flow.requires)) def test_get_cert_backup_rest_anti_affinity_create_amphora_for_lb_flow( self, mock_get_net_driver): @@ -208,7 +208,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.SERVER_PEM, amp_flow.provides) self.assertEqual(5, len(amp_flow.provides)) - self.assertEqual(3, len(amp_flow.requires)) + self.assertEqual(4, len(amp_flow.requires)) self.conf.config(group="nova", enable_anti_affinity=False) def test_get_delete_amphora_flow(self, mock_get_net_driver): @@ -259,7 +259,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.LISTENERS, amp_flow.provides) self.assertIn(constants.LOADBALANCER, amp_flow.provides) - self.assertEqual(3, len(amp_flow.requires)) + self.assertEqual(4, len(amp_flow.requires)) self.assertEqual(12, len(amp_flow.provides)) amp_flow = self.AmpFlow.get_failover_flow( @@ -279,7 +279,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.LISTENERS, amp_flow.provides) self.assertIn(constants.LOADBALANCER, amp_flow.provides) - self.assertEqual(3, len(amp_flow.requires)) + self.assertEqual(4, len(amp_flow.requires)) self.assertEqual(12, len(amp_flow.provides)) amp_flow = self.AmpFlow.get_failover_flow( @@ -299,7 +299,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.LISTENERS, amp_flow.provides) self.assertIn(constants.LOADBALANCER, amp_flow.provides) - self.assertEqual(3, len(amp_flow.requires)) + self.assertEqual(4, len(amp_flow.requires)) self.assertEqual(12, len(amp_flow.provides)) amp_flow = self.AmpFlow.get_failover_flow( @@ -319,7 +319,7 @@ class TestAmphoraFlows(base.TestCase): self.assertIn(constants.LISTENERS, amp_flow.provides) self.assertIn(constants.LOADBALANCER, amp_flow.provides) - self.assertEqual(3, len(amp_flow.requires)) + self.assertEqual(4, len(amp_flow.requires)) self.assertEqual(12, len(amp_flow.provides)) def test_get_failover_flow_spare(self, mock_get_net_driver): diff --git a/octavia/tests/unit/controller/worker/flows/test_load_balancer_flows.py b/octavia/tests/unit/controller/worker/flows/test_load_balancer_flows.py index 9d302b6fcd..eb069b1d82 100644 --- a/octavia/tests/unit/controller/worker/flows/test_load_balancer_flows.py +++ b/octavia/tests/unit/controller/worker/flows/test_load_balancer_flows.py @@ -213,7 +213,7 @@ class TestLoadBalancerFlows(base.TestCase): self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, create_flow.provides) - self.assertEqual(3, len(create_flow.requires)) + self.assertEqual(4, len(create_flow.requires)) self.assertEqual(12, len(create_flow.provides), create_flow.provides) @@ -241,6 +241,6 @@ class TestLoadBalancerFlows(base.TestCase): self.assertIn(constants.AMPHORAE_NETWORK_CONFIG, create_flow.provides) - self.assertEqual(3, len(create_flow.requires)) + self.assertEqual(4, len(create_flow.requires)) self.assertEqual(12, len(create_flow.provides), create_flow.provides) diff --git a/octavia/tests/unit/controller/worker/test_controller_worker.py b/octavia/tests/unit/controller/worker/test_controller_worker.py index c2c926a379..f474a10c09 100644 --- a/octavia/tests/unit/controller/worker/test_controller_worker.py +++ b/octavia/tests/unit/controller/worker/test_controller_worker.py @@ -1142,6 +1142,8 @@ class TestControllerWorker(base.TestCase): _flow_mock.run.assert_called_once_with() + @mock.patch('octavia.db.repositories.FlavorRepository.' + 'get_flavor_metadata_dict', return_value={}) @mock.patch('octavia.controller.worker.flows.' 'amphora_flows.AmphoraFlows.get_failover_flow', return_value=_flow_mock) @@ -1149,6 +1151,7 @@ class TestControllerWorker(base.TestCase): def test_failover_amphora(self, mock_update, mock_get_failover_flow, + mock_get_flavor_meta, mock_api_get_session, mock_dyn_log_listener, mock_taskflow_load, @@ -1173,7 +1176,8 @@ class TestControllerWorker(base.TestCase): constants.LOADBALANCER_ID: _amphora_mock.load_balancer_id, constants.BUILD_TYPE_PRIORITY: - constants.LB_CREATE_FAILOVER_PRIORITY + constants.LB_CREATE_FAILOVER_PRIORITY, + constants.FLAVOR: {} })) _flow_mock.run.assert_called_once_with() @@ -1329,6 +1333,8 @@ class TestControllerWorker(base.TestCase): mock_update.assert_called_with(_db_session, 123, provisioning_status=constants.ERROR) + @mock.patch('octavia.db.repositories.FlavorRepository.' + 'get_flavor_metadata_dict', return_value={}) @mock.patch('octavia.controller.worker.flows.' 'amphora_flows.AmphoraFlows.get_failover_flow', return_value=_flow_mock) @@ -1338,8 +1344,9 @@ class TestControllerWorker(base.TestCase): @mock.patch('octavia.db.repositories.LoadBalancerRepository.update') def test_failover_amphora_anti_affinity(self, mock_update, - mock_get_update_listener_flow, mock_get_lb_for_amphora, + mock_get_update_listener_flow, + mock_get_flavor_meta, mock_api_get_session, mock_dyn_log_listener, mock_taskflow_load, @@ -1368,6 +1375,7 @@ class TestControllerWorker(base.TestCase): constants.BUILD_TYPE_PRIORITY: constants.LB_CREATE_FAILOVER_PRIORITY, constants.SERVER_GROUP_ID: "123", + constants.FLAVOR: {} })) _flow_mock.run.assert_called_once_with() diff --git a/releasenotes/notes/add-compute-flavor-capability-ab202697a7fbdc3d.yaml b/releasenotes/notes/add-compute-flavor-capability-ab202697a7fbdc3d.yaml new file mode 100644 index 0000000000..aab61ac5b0 --- /dev/null +++ b/releasenotes/notes/add-compute-flavor-capability-ab202697a7fbdc3d.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Operators can now use the 'compute_flavor' Octavia flavor capability when + using the amphora provider driver. This allows custom compute driver + flavors to be used per-load balancer. If this is not defined in an + Octavia flavor, the amp_flavor_id Octavia configuration file setting + will continue to be used.