First dev tools -- Nailgun's container commands

Change-Id: I4ae770acf7d20510bf3020ebeaffeef8decd6c4f
This commit is contained in:
Przemyslaw Kaminski 2015-03-31 15:50:22 +02:00
parent 306828d69a
commit 1db69a76e4
15 changed files with 842 additions and 16 deletions

View File

@ -0,0 +1,309 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import logging
import os
import six
from cliff import command
from fuel_dev_tools import exc
from fuel_dev_tools import rsync
from fuel_dev_tools import ssh
DOCKER_CONTAINER_PATH = '/var/lib/docker/containers/'
DOCKER_DEVICEMAPPER_PATH = '/var/lib/docker/devicemapper/mnt/'
# TODO(pkaminski)
print_verbose = six.print_
class DockerMixin(object):
container = None
def get_container_config(self):
d = self.get_container_config_directory()
config = self.ssh_command('cat %s/config.json' % d).decode('utf-8')
return json.loads(config)
def get_container_config_directory(self):
iid = self.get_docker_id()
paths = self.ssh_command(
'ls %s | grep %s' % (DOCKER_CONTAINER_PATH, iid)
).decode('utf-8')
return os.path.join(DOCKER_CONTAINER_PATH, paths.split()[0])
def get_container_directory(self):
iid = self.get_docker_id()
paths = self.ssh_command(
'ls %s | grep %s' % (DOCKER_DEVICEMAPPER_PATH, iid)
).decode('utf-8')
return os.path.join(DOCKER_DEVICEMAPPER_PATH, paths.split()[0])
def get_docker_id(self, get_exited=False):
"""Returns first 12 characters of LXC container ID.
(as returned by the 'docker ps' command)
:param get_exited:
:return:
"""
up = self.ssh_command(
'docker ps -a | grep -i %s | grep Up | cut -f 1 -d " "' %
self.container
).decode('utf-8')
print_verbose('FOUND CONTAINERS: %r' % up)
if not up and get_exited:
print_verbose('Container not Up, trying Exited')
up = self.ssh_command(
'docker ps -a | grep -i %s | grep Exited | cut -f 1 -d " "' %
self.container
).decode('utf-8')
print_verbose('FOUND CONTAINERS: %r' % up)
if not up:
raise exc.DockerError(
"Container '%s' not found or not functional" %
self.container
)
return up
def get_full_docker_id(self, get_exited=False):
"""Returns full container ID.
:return:
"""
iid = self.get_docker_id(get_exited=get_exited)
iid = self.ssh_command(
"docker inspect -f '{{.ID}}' %s" % iid
).decode('utf-8').strip()
return iid
def get_log_files(self, args):
log_dir = self.get_log_directory()
files = '*.log'
if args.files:
if len(args.files) == 1:
files = '%s.log' % args.files[0]
else:
files = '{%s}.log' % ','.join(args.files)
return os.path.join(log_dir, files)
def restart_container(self):
result = self.ssh_command(
'docker restart %s' % self.get_docker_id()
)
print_verbose(result)
def start_container(self):
result = self.ssh_command(
'docker start %s' % self.get_docker_id(get_exited=True)
)
print_verbose(result)
def stop_container(self):
result = self.ssh_command(
'docker stop %s' % self.get_docker_id()
)
print_verbose(result)
class IdCommand(DockerMixin, ssh.SSHMixin, command.Command):
def take_action(self, parsed_args):
six.print_(self.get_docker_id(get_exited=True))
class ConfigCommand(DockerMixin, ssh.SSHMixin, command.Command):
def take_action(self, parsed_args):
six.print_(json.dumps(self.get_container_config(), indent=2))
class DirCommand(DockerMixin, ssh.SSHMixin, command.Command):
def take_action(self, parsed_args):
six.print_(self.get_container_directory())
class LogCommand(DockerMixin, ssh.SSHMixin, command.Command):
def get_log_directory(self):
raise NotImplementedError('No log directory for this command')
def get_parser(self, prog_name):
parser = super(LogCommand, self).get_parser(prog_name)
parser.add_argument(
'files',
type=str,
nargs='*',
help='List of files to show (all by default).'
)
return parser
def take_action(self, parsed_args):
six.print_(
self.ssh_command(
'tail', '-n', '100000', self.get_log_files(parsed_args)
)
)
class RestartCommand(DockerMixin, ssh.SSHMixin, command.Command):
def take_action(self, parsed_args):
self.restart_container()
class RsyncCommand(rsync.RsyncMixin,
DockerMixin,
ssh.SSHMixin,
command.Command):
def pre_sync(self, parsed_args):
pass
def post_sync(self, parsed_args):
pass
@property
def source_path(self):
return ''
@property
def target_path(self):
return ''
def get_parser(self, prog_name):
parser = super(RsyncCommand, self).get_parser(prog_name)
parser.add_argument(
'-s', '--source',
nargs='?',
default='.',
help='Source of the rsync-ed directory.'
)
return parser
def take_action(self, parsed_args):
self.pre_sync(parsed_args)
source_dir = parsed_args.source
base_target_dir = os.path.join(
self.get_container_directory(),
'rootfs'
)
source = os.path.join(source_dir, self.source_path)
# target is on the remote
target = os.path.join(base_target_dir, self.target_path)
self.rsync(source, target)
self.post_sync(parsed_args)
class ShellCommand(DockerMixin, ssh.SSHMixin, command.Command):
default_command = None
log = logging.getLogger(__name__)
def get_parser(self, prog_name):
parser = super(ShellCommand, self).get_parser(prog_name)
help_msg = 'Command to execute'
if self.default_command:
help_msg = '%s (default: %s)' % (help_msg, self.default_command)
parser.add_argument(
'-c', '--command',
default=None,
help=help_msg
)
return parser
def take_action(self, parsed_args):
command = parsed_args.command
if not command:
command = self.default_command
if not command:
command = '/bin/bash'
return self.ssh_command_interactive(
'lxc-attach', '--name', self.get_full_docker_id(), command
)
class StartCommand(DockerMixin, ssh.SSHMixin, command.Command):
def take_action(self, parsed_args):
self.start_container()
class StopCommand(DockerMixin, ssh.SSHMixin, command.Command):
def take_action(self, parsed_args):
self.stop_container()
class TailCommand(DockerMixin, ssh.SSHMixin, command.Command):
def get_log_directory(self):
raise NotImplementedError('No log directory for this command')
def get_parser(self, prog_name):
parser = super(TailCommand, self).get_parser(prog_name)
parser.add_argument(
'files',
type=str,
nargs='*',
help='List of files to show (all by default).'
)
return parser
def take_action(self, parsed_args):
self.ssh_command_interactive(
'tail', '-F', self.get_log_files(parsed_args)
)
class VolumesCommand(DockerMixin, ssh.SSHMixin, command.Command):
def take_action(self, parsed_args):
six.print_(
json.dumps(
self.get_container_config().get('Volumes', {}), indent=2
)
)

View File

@ -0,0 +1,87 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuel_dev_tools import docker
from fuel_dev_tools import info
class DockerNailgunMixin(object):
container = 'nailgun'
default_command = 'python'
def get_log_directory(self):
return '/var/log/docker-logs/nailgun'
class NailgunInfo(DockerNailgunMixin, info.BasicInfo):
@classmethod
def get_info(cls):
return ('If you login to the shell you have the possibility to '
'reset the database with commands\n'
'/usr/bin/nailgun_syncdb\n'
'/usr/bin/naligun_fixtures')
class Id(DockerNailgunMixin, docker.IdCommand):
"""Print Docker container id."""
class Config(DockerNailgunMixin, docker.ConfigCommand):
"""Print Docker container config."""
class Dir(DockerNailgunMixin, docker.DirCommand):
"""Print Docker container directory on master."""
class Log(DockerNailgunMixin, docker.LogCommand):
"""Display logs for container."""
class Restart(DockerNailgunMixin, docker.RestartCommand):
"""Restart Docker container."""
class Rsync(DockerNailgunMixin, docker.RsyncCommand):
"""Rsync local directory to the Docker container."""
@property
def source_path(self):
return 'nailgun/nailgun'
@property
def target_path(self):
return 'usr/lib/python2.6/site-packages/nailgun'
def post_sync(self, parsed_args):
self.restart_container()
class Start(DockerNailgunMixin, docker.StartCommand):
"""Start Docker container."""
class Stop(DockerNailgunMixin, docker.StopCommand):
"""Stop Docker container."""
class Shell(DockerNailgunMixin, docker.ShellCommand):
"""Shell into a nailgun Docker container."""
class Tail(DockerNailgunMixin, docker.TailCommand):
"""Display logs for container."""
class Volumes(DockerNailgunMixin, docker.VolumesCommand):
"""Print all volumes of a container."""

View File

@ -0,0 +1,80 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import six
from fabric import api as fabric_api
from fuel_dev_tools import docker
class DockerNginxMixin(object):
container = 'nginx'
default_command = '/bin/bash'
def get_log_directory(self):
return '/var/log/docker-logs/nginx'
class Rsync(DockerNginxMixin, docker.RsyncCommand):
"""Rsync static files to the Docker container."""
temporary_build_dir = 'built-static'
def get_parser(self, prog_name):
parser = super(Rsync, self).get_parser(prog_name)
parser.add_argument(
'--no-gulp',
action='store_true',
help=('Don\'t run Gulp building task (default: false; note that '
'by default the minified version is used.')
)
return parser
def take_action(self, parsed_args):
source_dir = parsed_args.source
# NOTE: slash at the end is important in source_path!
source_path = 'nailgun/%s/' % self.temporary_build_dir
if not parsed_args.no_gulp:
self.build_gulp_static(source_dir)
config = self.get_container_config()
target_dir = config['Volumes']['/usr/share/nailgun/static']
source = os.path.join(source_dir, source_path)
self.rsync(source, target_dir)
def build_gulp_static(self, source_dir):
cwd = os.path.join(source_dir, 'nailgun')
six.print_(
'Building gulp static in %s, temporary static dir is: %s...' % (
cwd,
self.temporary_build_dir
)
)
with fabric_api.lcd(cwd):
result = fabric_api.run(
fabric_api.local(
'gulp build --static-dir=%s' % self.temporary_build_dir
)
)
six.print_(result)

25
fuel_dev_tools/exc.py Normal file
View File

@ -0,0 +1,25 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
class ClientException(Exception):
pass
class DockerError(Exception):
pass
class SSHError(Exception):
pass

58
fuel_dev_tools/info.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import six
from cliff import command
class BasicInfo(object):
pass
class Info(command.Command):
"""Various useful information about the Fuel master node."""
log = logging.getLogger(__name__)
def take_action(self, parsed_args):
urls = [
('OpenStackAPI', 5000,
'http://developer.openstack.org/api-ref.html'),
('RabbitMQ', 5672, 'User/password: <empty>'),
('RabbitMQ Management', 15672, 'User/password: <empty>'),
]
six.print_('URLS:')
for name, port, info in urls:
six.print_('{name:{fill}{align}{width}}http://{IP}:{port:{fill}'
'{align}{width}}{info}'.format(
name=name,
IP=self.app_args.IP,
port=port,
info=info,
fill=' ',
width=20,
align='<')
)
classes = BasicInfo.__subclasses__()
for klass in sorted(classes, key=lambda k: k.__name__):
six.print_('-' * 20)
six.print_(klass.container.title())
if hasattr(klass, 'get_info'):
six.print_(klass.get_info())
six.print_('')

29
fuel_dev_tools/rsync.py Normal file
View File

@ -0,0 +1,29 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
from fabric.contrib import project
class RsyncMixin(object):
def rsync(self, source, target):
six.print_('RSYNC: %s --> %s' % (source, target))
result = project.rsync_project(
local_dir=source,
remote_dir=target
)
six.print_(result.decode('utf-8'))

118
fuel_dev_tools/shell.py Normal file
View File

@ -0,0 +1,118 @@
#!/usr/bin/env python
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Command-line utility to help developers work with Fuel master.
"""
import logging
import os
from oslo.utils import encodeutils
import six
import sys
import traceback
from cliff import app
from cliff import commandmanager
from docker import nailgun
from docker import nginx
import exc
import info
COMMANDS = {
'info': info.Info,
'nailgun-id': nailgun.Id,
'nailgun-config': nailgun.Config,
'nailgun-dir': nailgun.Dir,
'nailgun-log': nailgun.Log,
'nailgun-restart': nailgun.Restart,
'nailgun-rsync': nailgun.Rsync,
'nailgun-rsync-static': nginx.Rsync,
'nailgun-shell': nailgun.Shell,
'nailgun-start': nailgun.Start,
'nailgun-stop': nailgun.Stop,
'nailgun-tail': nailgun.Tail,
'nailgun-volumes': nailgun.Volumes,
}
class ToolsApp(app.App):
log = logging.getLogger(__name__)
def __init__(self):
super(ToolsApp, self).__init__(
description=__doc__.strip(),
version='1.0',
command_manager=commandmanager.CommandManager('fuel.cli')
)
self.commands = COMMANDS
for k, v in self.commands.items():
self.command_manager.add_command(k, v)
def build_option_parser(self, description, version):
"""Return an argparse option parser for this application.
Subclasses may override this method to extend
the parser with more global options.
:param description: full description of the application
:paramtype description: str
:param version: version number for the application
:paramtype version: str
"""
parser = super(ToolsApp, self).build_option_parser(
description, version)
parser.add_argument(
'--IP',
default='10.20.0.2',
help='Fuel master node IP address'
)
parser.add_argument(
'-I', '--identity-file',
default=os.path.join(
os.environ['HOME'], '.ssh', 'id_rsa.openstack'),
help='SSH identity file'
)
return parser
def main(argv=sys.argv[1:]):
try:
return ToolsApp().run(
list(map(encodeutils.safe_decode, argv))
)
except KeyboardInterrupt:
six.print_("... terminating client", file=sys.stderr)
return 130
except exc.ClientException:
six.print_('ClientException')
return 1
except exc.SSHError:
six.print_('SSHError')
return 1
except Exception:
traceback.print_exc()
return 1
if __name__ == '__main__':
main()

109
fuel_dev_tools/ssh.py Normal file
View File

@ -0,0 +1,109 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
import subprocess
import fabric
from fabric import api as fabric_api
import exc
SSH_PASSWORD_CHECKED = False
print_verbose = six.print_
# TODO(pkaminski): ssh_command should be in some utils, not necessarily
# in this class?
class SSHMixin(object):
def __init__(self, *args, **kwargs):
super(SSHMixin, self).__init__(*args, **kwargs)
if self.app_args:
fabric_api.env.host_string = self.app_args.IP
fabric_api.env.user = 'root'
if not self.app_args.debug:
for key in fabric.state.output:
fabric.state.output[key] = False
def send_identity(self):
print_verbose('Sending identity %s for passwordless authentication' %
self.app_args.identity_file)
with open('%s.pub' % self.app_args.identity_file) as f:
contents = f.read()
result = fabric_api.run(
"echo '%s' >> ~/.ssh/authorized_keys" % contents
)
print_verbose(result)
# And while we're here, let's fix /etc/hosts for which 10.20.0.2
# points to some non-existing domain (with misconfigured reverse-DNS
# lookups each SSH connection can be quite slow)
result = fabric_api.run(
"sed -i 's/^%(IP)s.*/%(IP)s localhost/' /etc/hosts" % {
'IP': self.app_args.IP
}
)
print_verbose(result)
# Need to restart after /etc/hosts change
result = fabric_api.run('service sshd restart')
print_verbose(result)
return result
def ssh_command(self, *args):
global SSH_PASSWORD_CHECKED
if not SSH_PASSWORD_CHECKED:
# NOTE: test if key is added to .authorized_keys with
SSH_PASSWORD_CHECKED = True
try:
subprocess.check_output([
'ssh',
'-o',
'PasswordAuthentication=no',
'root@%s' % self.app_args.IP,
'echo 1'
], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
if 'remote host identification has changed' in \
six.u(e.output).decode('utf-8').lower():
# .ssh/known_hosts error
raise exc.SSHError(e.output)
# Exit code error -- send .pub key to host
self.send_identity()
return fabric_api.run(' '.join(args))
def ssh_command_interactive(self, *args):
print_verbose("COMMAND: %r" % list(args))
command = None
if args:
command = ' '.join(args)
print_verbose('interactive', command)
fabric_api.open_shell(command=command)

View File

@ -13,16 +13,16 @@
# under the License.
"""
test_fuel-dev-tools
test_fuel_dev_tools
----------------------------------
Tests for `fuel-dev-tools` module.
Tests for `fuel_dev_tools` module.
"""
from fuel-dev-tools.tests import base
from fuel_dev_tools.tests import base
class TestFuel-dev-tools(base.TestCase):
class TestFuelDevTools(base.TestCase):
def test_something(self):
pass

View File

@ -4,3 +4,7 @@
pbr>=0.6,!=0.7,<1.0
Babel>=1.3
cliff>=1.10.1
oslo.utils>=1.4.0
fabric>=1.10.1

View File

@ -22,7 +22,11 @@ classifier =
[files]
packages =
fuel-dev-tools
fuel_dev_tools
[entry_points]
console_scripts =
fuel-dev-tools = fuel_dev_tools.shell:main
[build_sphinx]
source-dir = doc/source
@ -33,15 +37,15 @@ all_files = 1
upload-dir = doc/build/html
[compile_catalog]
directory = fuel-dev-tools/locale
domain = fuel-dev-tools
directory = fuel_dev_tools/locale
domain = fuel_dev_tools
[update_catalog]
domain = fuel-dev-tools
output_dir = fuel-dev-tools/locale
input_file = fuel-dev-tools/locale/fuel-dev-tools.pot
domain = fuel_dev_tools
output_dir = fuel_dev_tools/locale
input_file = fuel_dev_tools/locale/fuel-dev-tools.pot
[extract_messages]
keywords = _ gettext ngettext l_ lazy_gettext
mapping_file = babel.cfg
output_file = fuel-dev-tools/locale/fuel-dev-tools.pot
output_file = fuel_dev_tools/locale/fuel-dev-tools.pot

13
tox.ini
View File

@ -1,6 +1,6 @@
[tox]
minversion = 1.6
envlist = py33,py34,py26,py27,pypy,pep8
minversion = 1.8
envlist = py26,py27,pep8
skipsdist = True
[testenv]
@ -13,7 +13,10 @@ deps = -r{toxinidir}/requirements.txt
commands = python setup.py testr --slowest --testr-args='{posargs}'
[testenv:pep8]
commands = flake8
deps = hacking==0.10.1
usedevelop = False
commands =
flake8
[testenv:venv]
commands = {posargs}
@ -29,8 +32,8 @@ commands = oslo_debug_helper {posargs}
[flake8]
# E123, E125 skipped as they are invalid PEP-8.
count = True
show-source = True
ignore = E123,E125
builtins = _
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build
exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,vagrant/*