x
/
kolla-mesos
Code Issues Proposed changes

Cleanup volumes on multiple nodes 

"For development only" script removing containers and volumes
via SSH. Temporary thing before we switch to persistent volumes
offered by Mesos.

Change-Id: I7cf4a33021ebe8a1953b42ad39384e7c9c96eaae
Partially-Implements: blueprint multinode
This commit is contained in:
Michal Rostecki 2016-03-10 12:16:28 +01:00 committed by Michal Rostecki
parent cacdf3fb28
commit f04bce6a26
9 changed files with 94 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
kolla_mesos/cleanup.py
View File

@ -10,18 +10,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
import multiprocessing
import operator
import re
import contextlib
from oslo_config import cfg
from oslo_log import log as logging
import paramiko
import retrying
import six
from kolla_mesos import chronos
from kolla_mesos.common import docker_utils
from kolla_mesos.common import mesos_utils
from kolla_mesos.common import zk_utils
from kolla_mesos import exception
from kolla_mesos import marathon
@ -29,10 +26,30 @@ from kolla_mesos import mesos
CONF = cfg.CONF
CONF.import_group('ssh', 'kolla_mesos.config.ssh')
LOG = logging.getLogger(__name__)
@contextlib.contextmanager
def ssh_conn(hostname):
ssh_client = paramiko.client.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
LOG.info("Establishing SSH connection to %s", hostname)
ssh_client.connect(hostname, username=CONF.ssh.username)
yield ssh_client
ssh_client.close()
def execute_command(ssh_client, command):
_, stdout, stderr = ssh_client.exec_command(command)
# We have to read the stdout and stderr to ensure that we'll not do
# anything before command ends its execution.
stdout.read()
stderr.read()
@retrying.retry(wait_fixed=5000)
def wait_for_mesos_cleanup():
"""Check whether all tasks in Mesos are exited."""
@ -44,83 +61,6 @@ def wait_for_mesos_cleanup():
raise exception.MesosTasksNotCompleted()
@docker_utils.DockerClient()
def remove_container(dc, container_name):
LOG.info("Removing container %s", container_name)
dc.remove_container(container_name)
# NOTE(nihilifer): Despite the fact that OpenStack community decided to use
# builtins like "map", "filter" etc. directly, without aiming to use lazy
# generators in Python 2.x, here we decided to always use generators in every
# version of Python. Mainly because Mesos cluster may have a lot of containers
# and we would do multiple O(n) operations. Doing all these things lazy
# results in iterating only once on the lists of containers and volumes.
def get_container_names():
with docker_utils.DockerClient() as dc:
exited_containers = dc.containers(all=True,
filters={'status': 'exited'})
created_containers = dc.containers(all=True,
filters={'status': 'created'})
dead_containers = dc.containers(all=True,
filters={'status': 'dead'})
containers = itertools.chain(exited_containers, created_containers,
dead_containers)
container_name_lists = six.moves.map(operator.itemgetter('Names'),
containers)
container_name_lists = six.moves.filter(lambda name_list:
len(name_list) > 0,
container_name_lists)
container_names = six.moves.map(operator.itemgetter(0),
container_name_lists)
container_names = six.moves.filter(lambda name: re.search(r'/mesos-',
name),
container_names)
return container_names
# NOTE(nihilifer): Mesos doesn't support fully the named volumes which we're
# using. Mesos can run containers with named volume with passing the Docker
# parameters directly, but it doesn't handle any other actions with them.
# That's why currently we're cleaning the containers and volumes by calling
# the Docker API directly.
# TODO(nihilifer): Request/develop the feature of cleaning volumes directly
# in Mesos and Marathon.
# TODO(nihilifer): Support multinode cleanup.
def remove_all_containers():
"""Remove all exited containers which were run by Mesos.
It's done in order to succesfully remove named volumes.
"""
container_names = get_container_names()
# Remove containers in the pool of workers
pool = multiprocessing.Pool(processes=CONF.workers)
tasks = [pool.apply_async(remove_container, (container_name,))
for container_name in container_names]
# Wait for every task to execute
for task in tasks:
task.get()
@docker_utils.DockerClient()
def remove_all_volumes(dc):
"""Remove all volumes created for containers run by Mesos."""
if dc.volumes()['Volumes'] is not None:
volume_names = six.moves.map(operator.itemgetter('Name'),
dc.volumes()['Volumes'])
for volume_name in volume_names:
# TODO(nihilifer): Provide a more intelligent filtering for Mesos
# infra volumes.
if 'zookeeper' not in volume_name:
LOG.info("Removing volume %s", volume_name)
dc.remove_volume(volume_name)
else:
LOG.info("No docker volumes found")
def cleanup():
LOG.info("Starting cleanup...")
marathon_client = marathon.Client()
@ -138,7 +78,20 @@ def cleanup():
LOG.info("Checking whether all tasks in Mesos are exited")
wait_for_mesos_cleanup()
LOG.info("Starting cleanup of Docker containers")
remove_all_containers()
LOG.info("Starting cleanup of Docker volumes")
remove_all_volumes()
hostnames = mesos_utils.get_slave_hostnames()
for hostname in hostnames:
with ssh_conn(hostname) as ssh_client:
LOG.info("Removing all containers on host %s", hostname)
execute_command(
ssh_client,
'sudo docker rm -f -v $(docker ps -a --format "{{ .Names }}" '
'| grep mesos-)')
execute_command(
ssh_client,
'while sudo docker ps -a --format "{{ .Names }}" | grep '
'mesos-; do sleep 1; done')
LOG.info("Removing all named volumes on host %s", hostname)
execute_command(
ssh_client,
"sudo docker volume rm $(sudo docker volume ls -q | grep -v "
"zookeeper)")

7
kolla_mesos/common/mesos_utils.py
View File

@ -68,3 +68,10 @@ def get_marathon(mesos_client):
else:
marathon_framework = None
return marathon_framework
@MesosClient()
def get_slave_hostnames(mesos_client):
slaves = mesos_client.get_slaves()
hostnames = map(operator.itemgetter('hostname'), slaves)
return hostnames

2
kolla_mesos/common/type_utils.py
View File

@ -14,6 +14,6 @@
def str_to_bool(text):
if not text:
return False
if text.lower() in ['true', 'yes']:
if text.lower() in ['true', 'yes', 'y']:
return True
return False

30
kolla_mesos/common/docker_utils.py → kolla_mesos/common/warning_utils.py
View File

@ -11,22 +11,22 @@
# limitations under the License.
import functools
import sys
import docker
import six
from kolla_mesos.common import type_utils
class DockerClient(object):
"""Decorator and contextmanager for providing the Docker connection."""
def __enter__(self):
self.dc = docker.Client()
return self.dc
def __exit__(self, *args, **kwargs):
self.dc.close()
def __call__(self, f):
def yes_no_prompt(msg):
def wrapper(f):
@functools.wraps(f)
def wrapper(*args, **kwargs):
with self as dc:
return f(dc, *args, **kwargs)
return wrapper
def wrapped(*args, **kwargs):
full_msg = '%s [y/N] ' % msg
yes_no = six.moves.input(full_msg)
yes_no = type_utils.str_to_bool(yes_no)
if not yes_no:
sys.exit(1)
return f(*args, **kwargs)
return wrapped
return wrapper

28
kolla_mesos/config/ssh.py Normal file
View File

@ -0,0 +1,28 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from oslo_config import cfg
CONF = cfg.CONF
ssh_opts = [
cfg.StrOpt('username',
default=os.getlogin(),
help='Username to used in cleanup SSH connections')
]
ssh_opt_group = cfg.OptGroup(name='ssh',
title='Options for cleanup SSH connections')
CONF.register_group(ssh_opt_group)
CONF.register_cli_opts(ssh_opts, ssh_opt_group)
CONF.register_opts(ssh_opts, ssh_opt_group)

2
kolla_mesos/opts.py
View File

@ -17,6 +17,7 @@ from kolla_mesos.config import marathon
from kolla_mesos.config import mesos
from kolla_mesos.config import network
from kolla_mesos.config import profiles
from kolla_mesos.config import ssh
from kolla_mesos.config import zookeeper
@ -27,6 +28,7 @@ def list_opts():
('marathon', marathon.marathon_opts),
('network', network.network_opts),
('profiles', profiles.profiles_opts),
('ssh', ssh.ssh_opts),
('zookeeper', zookeeper.zookeeper_opts),
('mesos', mesos.mesos_opts),
('', logging.logging_opts)

37
kolla_mesos/tests/common/test_docker_utils.py
View File

@ -1,37 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from kolla_mesos.common import docker_utils
from kolla_mesos.tests import base
@mock.patch('kolla_mesos.common.docker_utils.docker')
class TestDockerUtils(base.BaseTestCase):
def _asserts(self, docker_mock):
docker_mock.Client.assert_called_once_with()
docker_mock.Client().close.assert_called_once_with()
def test_contextmanager(self, docker_mock):
with docker_utils.DockerClient():
pass
self._asserts(docker_mock)
def test_decorator(self, docker_mock):
@docker_utils.DockerClient()
def decorated_function(dc):
return True
decorated_function()
self._asserts(docker_mock)

31
kolla_mesos/tests/test_cleanup.py
View File

@ -10,7 +10,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from oslo_config import cfg
import requests_mock
@ -46,33 +45,3 @@ class TestMesosCleanup(base.BaseTestCase):
json=MESOS_UNCLEAN_STATE)
self.assertRaises(exception.MesosTasksNotCompleted,
cleanup.wait_for_mesos_cleanup.__wrapped__)
@mock.patch('kolla_mesos.common.docker_utils.docker')
class TestDockerCleanup(base.BaseTestCase):
def setUp(self):
super(TestDockerCleanup, self).setUp()
CONF.set_override('workers', 1)
def test_remove_container(self, docker_mock):
cleanup.remove_container('test_container')
docker_mock.Client().remove_container.assert_called_once_with(
'test_container')
def test_get_container_names(self, docker_mock):
docker_mock.Client().containers.side_effect = [
[{'Names': ['/mesos-1']}, {'Names': ['/mesos-2']}],
[{'Names': ['/mesos-3']}], [{'Names': ['/mesos-4']}]
]
container_names = list(cleanup.get_container_names())
self.assertListEqual(['/mesos-1', '/mesos-2', '/mesos-3', '/mesos-4'],
container_names)
def test_remove_all_volumes(self, docker_mock):
docker_mock.Client().volumes.return_value = {'Volumes': [
{'Name': 'test_1'}, {'Name': 'test_2'}
]}
cleanup.remove_all_volumes()
docker_mock.Client().remove_volume.assert_has_calls(
[mock.call('test_1'), mock.call('test_2')])

1
requirements.txt
View File

@ -13,6 +13,7 @@ netifaces>=0.10.4 # MIT
oslo.config>=3.7.0 # Apache-2.0
oslo.utils>=3.5.0 # Apache-2.0
oslo.log>=1.14.0 # Apache-2.0
paramiko>=1.16.0 # LGPL
PrettyTable<0.8,>=0.7 # BSD
PyYAML>=3.1.0 # MIT
retrying!=1.3.0,>=1.2.3 # Apache-2.0