openstack
/
storlets
Code Issues Proposed changes

Allow limiting number of containers on a node 

This change introduces the new configuration option to limit number
of storlet containers running on a single now. This helps operators
avoid some nodes from being unevenly loaded.

To search storlet containers, The new container label,
 managed=by=storlets
is added to all storlet containers. This allows users to run additional
containers on a same node and exclude these containers from the limit.

Change-Id: I148fa5793c19e8f894b8cb3ccecf75b3d15fb17a
This commit is contained in:
Takashi Kajinami 2021-08-14 18:35:52 +09:00
parent 33c403e14b
commit a3583b598b
3 changed files with 58 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
etc/storlet-docker-gateway.conf-sample
View File

@ -8,3 +8,4 @@ pipes_dir = /home/docker_device/pipes/scopes
docker_repo = localhost:5001
restart_linux_container_timeout = 10
storlet_timeout = 40
max_containers_per_node = 0

14
storlets/gateway/gateways/docker/runtime.py
View File

@ -236,6 +236,9 @@ class RunTimeSandbox(object):
conf.get('default_docker_image_name',
'ubuntu_20.04_jre11_storlets')
self.max_containers_per_node = \
int(conf.get('max_containers_per_node', 0))
def ping(self):
"""
Ping to daemon factory process inside container
@ -319,11 +322,20 @@ class RunTimeSandbox(object):
else:
scontainer.stop(timeout=1)
# Check whether a new container can be started
if self.max_containers_per_node > 0:
all_scontainers = client.containers.list(
filters={'label': 'managed_by=storlets'})
if len(all_scontainers) >= self.max_containers_per_node:
raise StorletRuntimeException(
"Cannot start a container because of limit")
# Start the new one
client.containers.run(
docker_image_name, detach=True, name=docker_container_name,
network_disabled=True, mounts=mounts, user='swift',
auto_remove=True, stop_signal='SIGHUP')
auto_remove=True, stop_signal='SIGHUP',
labels={'managed_by': 'storlets'})
except docker.errors.ImageNotFound:
msg = "Image %s is not found" % docker_image_name
raise StorletRuntimeException(msg)

44
tests/unit/gateway/gateways/docker/test_runtime.py
View File

@ -313,6 +313,7 @@ class TestRunTimeSandbox(unittest.TestCase):
self.sbox._restart('storlet_image')
self.assertEqual(1, mock_containers.get.call_count)
self.assertEqual(0, mock_containers.list.call_count)
self.assertEqual(1, mock_containers.run.call_count)
# storlet container is running
@ -330,6 +331,7 @@ class TestRunTimeSandbox(unittest.TestCase):
self.sbox._restart('storlet_image')
self.assertEqual(1, mock_containers.get.call_count)
self.assertEqual(1, mock_container.stop.call_count)
self.assertEqual(0, mock_containers.list.call_count)
self.assertEqual(1, mock_containers.run.call_count)
# get failed
@ -366,6 +368,7 @@ class TestRunTimeSandbox(unittest.TestCase):
self.sbox._restart('storlet_image')
self.assertEqual(1, mock_containers.get.call_count)
self.assertEqual(1, mock_container.stop.call_count)
self.assertEqual(0, mock_containers.list.call_count)
self.assertEqual(0, mock_containers.run.call_count)
# run failed
@ -386,8 +389,49 @@ class TestRunTimeSandbox(unittest.TestCase):
self.sbox._restart('storlet_image')
self.assertEqual(1, mock_containers.get.call_count)
self.assertEqual(1, mock_container.stop.call_count)
self.assertEqual(0, mock_containers.list.call_count)
self.assertEqual(1, mock_containers.run.call_count)
# Set the limit
self.sbox.max_containers_per_node = 2
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'docker.from_env') as docker_from_env:
mock_client = mock.MagicMock(spec_set=docker.client.DockerClient)
mock_containers = mock.MagicMock(
spec_set=docker.models.containers.ContainerCollection)
mock_client.containers = mock_containers
mock_container = \
mock.MagicMock(spec_set=docker.models.containers.Container)
mock_containers.get.return_value = mock_container
mock_containers.list.return_value = [mock.MagicMock()]
docker_from_env.return_value = mock_client
self.sbox._restart('storlet_image')
self.assertEqual(1, mock_containers.get.call_count)
self.assertEqual(1, mock_container.stop.call_count)
self.assertEqual(1, mock_containers.list.call_count)
self.assertEqual(1, mock_containers.run.call_count)
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'docker.from_env') as docker_from_env:
mock_client = mock.MagicMock(spec_set=docker.client.DockerClient)
mock_containers = mock.MagicMock(
spec_set=docker.models.containers.ContainerCollection)
mock_client.containers = mock_containers
mock_container = \
mock.MagicMock(spec_set=docker.models.containers.Container)
mock_containers.get.return_value = mock_container
mock_containers.list.return_value = [mock.MagicMock()] * 2
docker_from_env.return_value = mock_client
with self.assertRaises(StorletRuntimeException):
self.sbox._restart('storlet_image')
self.assertEqual(1, mock_containers.get.call_count)
self.assertEqual(1, mock_container.stop.call_count)
self.assertEqual(1, mock_containers.list.call_count)
self.assertEqual(0, mock_containers.run.call_count)
def test_restart(self):
with mock.patch('storlets.gateway.gateways.docker.runtime.'
'RunTimePaths.create_host_pipe_dir') as pipe_dir, \