Allow limiting number of containers on a node
This change introduces the new configuration option to limit number of storlet containers running on a single now. This helps operators avoid some nodes from being unevenly loaded. To search storlet containers, The new container label, managed=by=storlets is added to all storlet containers. This allows users to run additional containers on a same node and exclude these containers from the limit. Change-Id: I148fa5793c19e8f894b8cb3ccecf75b3d15fb17a
This commit is contained in:
parent
33c403e14b
commit
a3583b598b
|
@ -8,3 +8,4 @@ pipes_dir = /home/docker_device/pipes/scopes
|
||||||
docker_repo = localhost:5001
|
docker_repo = localhost:5001
|
||||||
restart_linux_container_timeout = 10
|
restart_linux_container_timeout = 10
|
||||||
storlet_timeout = 40
|
storlet_timeout = 40
|
||||||
|
max_containers_per_node = 0
|
||||||
|
|
|
@ -236,6 +236,9 @@ class RunTimeSandbox(object):
|
||||||
conf.get('default_docker_image_name',
|
conf.get('default_docker_image_name',
|
||||||
'ubuntu_20.04_jre11_storlets')
|
'ubuntu_20.04_jre11_storlets')
|
||||||
|
|
||||||
|
self.max_containers_per_node = \
|
||||||
|
int(conf.get('max_containers_per_node', 0))
|
||||||
|
|
||||||
def ping(self):
|
def ping(self):
|
||||||
"""
|
"""
|
||||||
Ping to daemon factory process inside container
|
Ping to daemon factory process inside container
|
||||||
|
@ -319,11 +322,20 @@ class RunTimeSandbox(object):
|
||||||
else:
|
else:
|
||||||
scontainer.stop(timeout=1)
|
scontainer.stop(timeout=1)
|
||||||
|
|
||||||
|
# Check whether a new container can be started
|
||||||
|
if self.max_containers_per_node > 0:
|
||||||
|
all_scontainers = client.containers.list(
|
||||||
|
filters={'label': 'managed_by=storlets'})
|
||||||
|
if len(all_scontainers) >= self.max_containers_per_node:
|
||||||
|
raise StorletRuntimeException(
|
||||||
|
"Cannot start a container because of limit")
|
||||||
|
|
||||||
# Start the new one
|
# Start the new one
|
||||||
client.containers.run(
|
client.containers.run(
|
||||||
docker_image_name, detach=True, name=docker_container_name,
|
docker_image_name, detach=True, name=docker_container_name,
|
||||||
network_disabled=True, mounts=mounts, user='swift',
|
network_disabled=True, mounts=mounts, user='swift',
|
||||||
auto_remove=True, stop_signal='SIGHUP')
|
auto_remove=True, stop_signal='SIGHUP',
|
||||||
|
labels={'managed_by': 'storlets'})
|
||||||
except docker.errors.ImageNotFound:
|
except docker.errors.ImageNotFound:
|
||||||
msg = "Image %s is not found" % docker_image_name
|
msg = "Image %s is not found" % docker_image_name
|
||||||
raise StorletRuntimeException(msg)
|
raise StorletRuntimeException(msg)
|
||||||
|
|
|
@ -313,6 +313,7 @@ class TestRunTimeSandbox(unittest.TestCase):
|
||||||
|
|
||||||
self.sbox._restart('storlet_image')
|
self.sbox._restart('storlet_image')
|
||||||
self.assertEqual(1, mock_containers.get.call_count)
|
self.assertEqual(1, mock_containers.get.call_count)
|
||||||
|
self.assertEqual(0, mock_containers.list.call_count)
|
||||||
self.assertEqual(1, mock_containers.run.call_count)
|
self.assertEqual(1, mock_containers.run.call_count)
|
||||||
|
|
||||||
# storlet container is running
|
# storlet container is running
|
||||||
|
@ -330,6 +331,7 @@ class TestRunTimeSandbox(unittest.TestCase):
|
||||||
self.sbox._restart('storlet_image')
|
self.sbox._restart('storlet_image')
|
||||||
self.assertEqual(1, mock_containers.get.call_count)
|
self.assertEqual(1, mock_containers.get.call_count)
|
||||||
self.assertEqual(1, mock_container.stop.call_count)
|
self.assertEqual(1, mock_container.stop.call_count)
|
||||||
|
self.assertEqual(0, mock_containers.list.call_count)
|
||||||
self.assertEqual(1, mock_containers.run.call_count)
|
self.assertEqual(1, mock_containers.run.call_count)
|
||||||
|
|
||||||
# get failed
|
# get failed
|
||||||
|
@ -366,6 +368,7 @@ class TestRunTimeSandbox(unittest.TestCase):
|
||||||
self.sbox._restart('storlet_image')
|
self.sbox._restart('storlet_image')
|
||||||
self.assertEqual(1, mock_containers.get.call_count)
|
self.assertEqual(1, mock_containers.get.call_count)
|
||||||
self.assertEqual(1, mock_container.stop.call_count)
|
self.assertEqual(1, mock_container.stop.call_count)
|
||||||
|
self.assertEqual(0, mock_containers.list.call_count)
|
||||||
self.assertEqual(0, mock_containers.run.call_count)
|
self.assertEqual(0, mock_containers.run.call_count)
|
||||||
|
|
||||||
# run failed
|
# run failed
|
||||||
|
@ -386,8 +389,49 @@ class TestRunTimeSandbox(unittest.TestCase):
|
||||||
self.sbox._restart('storlet_image')
|
self.sbox._restart('storlet_image')
|
||||||
self.assertEqual(1, mock_containers.get.call_count)
|
self.assertEqual(1, mock_containers.get.call_count)
|
||||||
self.assertEqual(1, mock_container.stop.call_count)
|
self.assertEqual(1, mock_container.stop.call_count)
|
||||||
|
self.assertEqual(0, mock_containers.list.call_count)
|
||||||
self.assertEqual(1, mock_containers.run.call_count)
|
self.assertEqual(1, mock_containers.run.call_count)
|
||||||
|
|
||||||
|
# Set the limit
|
||||||
|
self.sbox.max_containers_per_node = 2
|
||||||
|
|
||||||
|
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||||
|
'docker.from_env') as docker_from_env:
|
||||||
|
mock_client = mock.MagicMock(spec_set=docker.client.DockerClient)
|
||||||
|
mock_containers = mock.MagicMock(
|
||||||
|
spec_set=docker.models.containers.ContainerCollection)
|
||||||
|
mock_client.containers = mock_containers
|
||||||
|
mock_container = \
|
||||||
|
mock.MagicMock(spec_set=docker.models.containers.Container)
|
||||||
|
mock_containers.get.return_value = mock_container
|
||||||
|
mock_containers.list.return_value = [mock.MagicMock()]
|
||||||
|
docker_from_env.return_value = mock_client
|
||||||
|
|
||||||
|
self.sbox._restart('storlet_image')
|
||||||
|
self.assertEqual(1, mock_containers.get.call_count)
|
||||||
|
self.assertEqual(1, mock_container.stop.call_count)
|
||||||
|
self.assertEqual(1, mock_containers.list.call_count)
|
||||||
|
self.assertEqual(1, mock_containers.run.call_count)
|
||||||
|
|
||||||
|
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||||
|
'docker.from_env') as docker_from_env:
|
||||||
|
mock_client = mock.MagicMock(spec_set=docker.client.DockerClient)
|
||||||
|
mock_containers = mock.MagicMock(
|
||||||
|
spec_set=docker.models.containers.ContainerCollection)
|
||||||
|
mock_client.containers = mock_containers
|
||||||
|
mock_container = \
|
||||||
|
mock.MagicMock(spec_set=docker.models.containers.Container)
|
||||||
|
mock_containers.get.return_value = mock_container
|
||||||
|
mock_containers.list.return_value = [mock.MagicMock()] * 2
|
||||||
|
docker_from_env.return_value = mock_client
|
||||||
|
|
||||||
|
with self.assertRaises(StorletRuntimeException):
|
||||||
|
self.sbox._restart('storlet_image')
|
||||||
|
self.assertEqual(1, mock_containers.get.call_count)
|
||||||
|
self.assertEqual(1, mock_container.stop.call_count)
|
||||||
|
self.assertEqual(1, mock_containers.list.call_count)
|
||||||
|
self.assertEqual(0, mock_containers.run.call_count)
|
||||||
|
|
||||||
def test_restart(self):
|
def test_restart(self):
|
||||||
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
with mock.patch('storlets.gateway.gateways.docker.runtime.'
|
||||||
'RunTimePaths.create_host_pipe_dir') as pipe_dir, \
|
'RunTimePaths.create_host_pipe_dir') as pipe_dir, \
|
||||||
|
|
Loading…
Reference in New Issue