Moved parts of ospluginutils and puppet plugin to installer core.

This is a preparation for refactoring of packstack.installer.setup_controller.Controller and packstack.installer.run_setup.
Code in this commit is not used yet

Change-Id: Iedb4c231a62e21536615f5ce9a44caaa88c8694f
This commit is contained in:
Martin Magr
2013-04-17 10:46:14 +02:00
parent 3d495d4415
commit 0e5addfda6
5 changed files with 641 additions and 0 deletions

View File

View File

@@ -0,0 +1,397 @@
# -*- coding: utf-8 -*-
import os
import stat
import uuid
import time
import logging
import tarfile
import tempfile
from .. import utils
class SshTarballTransferMixin(object):
"""
Transfers resources and recipes by packing them to tar.gz and
copying it via ssh.
"""
def _transfer(self, pack_path, pack_dest, res_dir):
node = self.node
args = locals()
# copy and extract tarball
script = utils.ScriptRunner()
script.append("scp %(pack_path)s root@%(node)s:%(pack_dest)s"
% args)
script.append("ssh -o StrictHostKeyChecking=no "
"-o UserKnownHostsFile=/dev/null root@%(node)s "
"tar -C %(res_dir)s -xpzf %(pack_dest)s" % args)
try:
script.execute()
except ScriptRuntimeError as ex:
# TO-DO: change to appropriate exception
raise RuntimeError('Failed to copy resources to node %s. '
'Reason: %s' % (node, ex))
def _pack_resources(self):
randpart = uuid.uuid4().hex[:8]
pack_path = os.path.join(self.local_tmpdir,
'res-%s.tar.gz' % randpart)
pack = tarfile.open(pack_path, mode='w:gz')
os.chmod(pack_path, stat.S_IRUSR | stat.S_IWUSR)
for path, dest in self._resources:
if not dest:
dest = os.path.basename(path)
pack.add(path,
arcname=os.path.join(dest, os.path.basename(path)))
pack.close()
return pack_path
def _copy_resources(self):
pack_path = self._pack_resources()
pack_dest = os.path.join(self.remote_tmpdir,
os.path.basename(pack_path))
self._transfer(pack_path, pack_dest, self.resource_dir)
def _pack_recipes(self):
randpart = uuid.uuid4().hex[:8]
pack_path = os.path.join(self.local_tmpdir,
'rec-%s.tar.gz' % randpart)
pack = tarfile.open(pack_path, mode='w:gz')
os.chmod(pack_path, stat.S_IRUSR | stat.S_IWUSR)
if self.recipe_dir.startswith(self.resource_dir):
dest = self.recipe_dir[len(self.resource_dir):].lstrip('/')
else:
dest = ''
for marker, recipes in self._recipes.iteritems():
for path in recipes:
_dest = os.path.join(dest, os.path.basename(path))
pack.add(path, arcname=_dest)
pack.close()
return pack_path
def _copy_recipes(self):
pack_path = self._pack_recipes()
pack_dest = os.path.join(self.remote_tmpdir,
os.path.basename(pack_path))
if self.recipe_dir.startswith(self.resource_dir):
extr_dest = self.resource_dir
else:
extr_dest = self.recipe_dir
self._transfer(pack_path, pack_dest, extr_dest)
class DroneObserver(object):
"""
Base class for listening messages from drones.
"""
def applying(self, drone, recipe):
"""
Drone is calling this method when it starts applying recipe.
"""
# subclass must implement this method
raise NotImplementedError()
def checking(self, drone, recipe):
"""
Drone is calling this method when it starts checking if recipe
has been applied.
"""
# subclass must implement this method
raise NotImplementedError()
def finished(self, drone, recipe):
"""
Drone is calling this method when it's finished with recipe
application.
"""
# subclass must implement this method
raise NotImplementedError()
class Drone(object):
"""
Base class used to apply installation recipes to nodes.
"""
def __init__(self, node, resource_dir=None, recipe_dir=None,
local_tmpdir=None, remote_tmpdir=None):
self._recipes = utils.SortedDict()
self._resources = []
self._applied = set()
self._running = set()
self._observer = None
# remote host IP or hostname
self.node = node
# working directories on remote host
self.resource_dir = resource_dir or \
'/tmp/drone%s' % uuid.uuid4().hex[:8]
self.recipe_dir = recipe_dir or \
os.path.join(self.resource_dir, 'recipes')
# temporary directories
self.remote_tmpdir = remote_tmpdir or \
'/tmp/drone%s' % uuid.uuid4().hex[:8]
self.local_tmpdir = local_tmpdir or \
tempfile.mkdtemp(prefix='drone')
def init_node(self):
"""
Initializes node for manipulation.
"""
created = []
server = utils.ScriptRunner(self.node)
for i in (self.resource_dir, self.recipe_dir,
self.remote_tmpdir):
server.append('mkdir -p %s' % os.path.dirname(i))
server.append('mkdir --mode 0700 %s' % i)
created.append('%s:%s' % (self.node, i))
server.execute()
# TO-DO: complete logger name when logging will be setup correctly
logger = logging.getLogger()
logger.debug('Created directories: %s' % ','.join(created))
@property
def recipes(self):
for i in self._recipes.itervalues():
for y in i:
yield y
@property
def resources(self):
for i in self._resources:
yield i[0]
def add_recipe(self, path, marker=None):
"""
Registers recipe for application on node. Recipes will be
applied in order they where added to drone. Multiple recipes can
be applied in paralel if they have same marker.
"""
marker = marker or uuid.uuid4().hex[:8]
self._recipes.setdefault(marker, []).append(path)
def add_resource(self, path, destination=None):
"""
Registers resource. Destination will be relative from resource
directory on node.
"""
dest = destination or ''
self._resources.append((path, dest))
def _copy_resources(self):
"""
Copies all local files registered in self._resources to their
appropriate destination on self.node. If tmpdir is given this
method can operate only in this directory.
"""
# subclass must implement this method
raise NotImplementedError()
def _copy_recipes(self):
"""
Copies all local files registered in self._recipes to their
appropriate destination on self.node. If tmpdir is given this
method can operate only in this directory.
"""
# subclass must implement this method
raise NotImplementedError()
def prepare_node(self):
"""
Copies all local resources and recipes to self.node.
"""
# TO-DO: complete logger name when logging will be setup correctly
logger = logging.getLogger()
logger.debug('Copying drone resources to node %s: %s'
% (self.node, self.resources))
self._copy_resources()
logger.debug('Copying drone recipes to node %s: %s'
% (self.node, [i[0] for i in self.recipes]))
self._copy_recipes()
def _apply(self, recipe):
"""
Starts application of single recipe given as path to the recipe
file in self.node. This method should not wait until recipe is
applied.
"""
# subclass must implement this method
raise NotImplementedError()
def _finished(self, recipe):
"""
Returns True if given recipe is applied, otherwise returns False
"""
# subclass must implement this method
raise NotImplementedError()
def _wait(self):
"""
Waits until all started applications of recipes will be finished
"""
while self._running:
_run = list(self._running)
for recipe in _run:
if self._observer:
self._observer.checking(self, recipe)
if self._finished(recipe):
self._applied.add(recipe)
self._running.remove(recipe)
if self._observer:
self._observer.finished(self, recipe)
else:
time.sleep(3)
continue
def set_observer(self, observer):
"""
Registers an observer. Given object should be subclass of class
DroneObserver.
"""
for attr in ('applying', 'checking', 'finished'):
if not hasattr(observer, attr):
raise ValueError('Observer object should be a subclass '
'of class DroneObserver.')
self._observer = observer
def apply(self, marker=None, name=None, skip=None):
"""
Applies recipes on node. If marker is specified, only recipes
with given marker are applied. If name is specified only recipe
with given name is applied. Skips recipes with names given
in list parameter skip.
"""
# TO-DO: complete logger name when logging will be setup correctly
logger = logging.getLogger()
skip = skip or []
lastmarker = None
for mark, recipelist in self._recipes.iteritems():
if marker and marker != mark:
logger.debug('Skipping marker %s for node %s.' %
(mark, self.node))
continue
for recipe in recipelist:
base = os.path.basename(recipe)
if (name and name != base) or base in skip:
logger.debug('Skipping recipe %s for node %s.' %
(recipe, self.node))
continue
# if the marker has changed then we don't want to
# proceed until all of the previous puppet runs have
# finished
if lastmarker and lastmarker != mark:
self._wait()
lastmarker = mark
logger.debug('Applying recipe %s to node %s.' %
(base, self.node))
rpath = os.path.join(self.recipe_dir, base)
if self._observer:
self._observer.applying(self, recipe)
self._running.add(rpath)
self._apply(rpath)
self._wait()
def cleanup(self, resource_dir=True, recipe_dir=True):
"""
Removes all directories created by this drone.
"""
shutil.rmtree(self.local_tmpdir, ignore_errors=True)
server = utils.ScriptRunner(self.node)
server.append('rm -fr %s' % self.remote_tmpdir)
if recipe_dir:
server.append('rm -fr %s' % self.recipe_dir)
if resource_dir:
server.append('rm -fr %s' % self.resource_dir)
server.execute()
class PackstackDrone(SshTarballTransferMixin, Drone):
"""
This drone uses Puppet and it's manifests to manipulate node.
"""
# XXX: Since this implementation is Packstack specific (_apply
# method), it should be moved out of installer when
# Controller and plugin system will be refactored and installer
# will support projects.
def __init__(self, *args, **kwargs):
kwargs['resource_dir'] = ('/var/tmp/packstack/drone%s'
% uuid.uuid4().hex[:8])
kwargs['recipe_dir'] = '%s/manifests' % kwargs['resource_dir']
kwargs['remote_tmpdir'] = '%s/temp' % kwargs['resource_dir']
super(PackstackDrone, self).__init__(*args, **kwargs)
self.module_dir = os.path.join(self.resource_dir, 'modules')
self.fact_dir = os.path.join(self.resource_dir, 'facts')
def init_node(self):
"""
Initializes node for manipulation.
"""
super(PackstackDrone, self).init_node()
server = utils.ScriptRunner(self.node)
for pkg in ("puppet", "openssh-clients", "tar"):
server.append("rpm -q %(pkg)s || yum install -y %(pkg)s"
% locals())
server.execute()
def add_resource(self, path, resource_type=None):
"""
Resource type should be module, fact or resource.
"""
resource_type = resource_type or 'resource'
dest = '%ss' % resource_type
super(PackstackDrone, self).add_resource(path, destination=dest)
def _finished(self, recipe):
recipe_base = os.path.basename(recipe)
log = os.path.join(self.recipe_dir,
recipe_base.replace(".finished", ".log"))
local = utils.ScriptRunner()
local.append('scp -o StrictHostKeyChecking=no '
'-o UserKnownHostsFile=/dev/null '
'root@%s:%s %s' % (self.node, recipe, log))
try:
# once a remote puppet run has finished, we retrieve
# the log file and check it for errors
local.execute(logerrors=False)
# if we got to this point the puppet apply has finished
return True
except utils.ScriptRuntimeError, e:
# the test raises an exception if the file doesn't exist yet
return False
def _apply(self, recipe):
running = "%s.running" % recipe
finished = "%s.finished" % recipe
server = utils.ScriptRunner(self.node)
server.append("touch %s" % running)
server.append("chmod 600 %s" % running)
# XXX: This is terrible hack, but unfortunatelly the apache
# puppet module doesn't work if we set FACTERLIB
# https://github.com/puppetlabs/puppetlabs-apache/pull/138
for bad_word in ('horizon', 'nagios', 'apache'):
if bad_word in recipe:
break
else:
server.append("export FACTERLIB=$FACTERLIB:%s" %
self.fact_dir)
server.append("export PACKSTACK_VAR_DIR=%s" % self.resource_dir)
# TO-DO: complete logger name when logging will be setup correctly
logger = logging.getLogger()
loglevel = logger.level <= logging.DEBUG and '--debug' or ''
rdir = self.resource_dir
mdir = self._module_dir
server.append(
"( flock %(rdir)s/ps.lock "
"puppet apply %(loglevel)s --modulepath %(mdir)s "
"%(recipe)s > %(running)s 2>&1 < /dev/null; "
"mv %(running)s %(finished)s ) "
"> /dev/null 2>&1 < /dev/null &" % locals())
server.execute()

View File

@@ -0,0 +1,214 @@
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013, Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import shutil
import tempfile
import subprocess
from unittest import TestCase
from ..test_base import PackstackTestCaseMixin
from packstack.installer.core.drones import *
from packstack.installer import utils
class SshTarballTransferMixinTestCase(PackstackTestCaseMixin, TestCase):
def setUp(self):
# Creating a temp directory that can be used by tests
self.tempdir = tempfile.mkdtemp()
def tearDown(self):
# remove the temp directory
#shutil.rmtree(self.tempdir)
pass
def setUp(self):
self.tempdir = tempfile.mkdtemp()
# prepare resource files
res1path = os.path.join(self.tempdir, 'res1.txt')
with open(res1path, 'w') as f:
f.write('resource one')
resdir = os.path.join(self.tempdir, 'resdir')
os.mkdir(resdir)
res2path = os.path.join(resdir, 'res2.txt')
with open(res2path, 'w') as f:
f.write('resource two')
# prepare recipe files
rec1path = os.path.join(self.tempdir, 'rec1.pp')
with open(rec1path, 'w') as f:
f.write('recipe one')
recdir = os.path.join(self.tempdir, 'recdir')
os.mkdir(recdir)
rec2path = os.path.join(recdir, 'rec2.pp')
with open(rec2path, 'w') as f:
f.write('recipe two')
# prepare class
self.mixin = SshTarballTransferMixin()
self.mixin.node = '127.0.0.1'
self.mixin.resource_dir = os.path.join(self.tempdir, 'remote')
self.mixin.recipe_dir = os.path.join(self.tempdir, 'remote',
'recipes')
self.mixin.local_tmpdir = os.path.join(self.tempdir, 'loctmp')
self.mixin.remote_tmpdir = os.path.join(self.tempdir, 'remtmp')
self.mixin._resources = [(res1path, 'resources'),
(resdir, 'resources')]
self.mixin._recipes = {'one': [rec1path], 'two': [rec2path]}
for i in (self.mixin.resource_dir, self.mixin.recipe_dir,
self.mixin.local_tmpdir, self.mixin.remote_tmpdir):
os.mkdir(i)
def test_tarball_packing(self):
"""
Tests packing of resources and recipes
"""
# pack
res_path = self.mixin._pack_resources()
rec_path = self.mixin._pack_recipes()
# unpack
for pack_path in (res_path, rec_path):
tarball = tarfile.open(pack_path)
tarball.extractall(path=self.tempdir)
# check content of files
for path, content in \
[('resources/res1.txt', 'resource one'),
('resources/resdir/res2.txt', 'resource two'),
('recipes/rec1.pp', 'recipe one'),
('recipes/rec2.pp', 'recipe two')]:
with open(os.path.join(self.tempdir, path)) as f:
fcont = f.read()
self.assertEqual(fcont, content)
# clean for next test
shutil.rmtree(os.path.join(self.tempdir, 'resources'))
shutil.rmtree(os.path.join(self.tempdir, 'recipes'))
'''
# uncomment this test only on local machines
def test_transfer(self):
"""
Tests resources transfer if sshd to 127.0.0.1 is enabled.
"""
remtmp = self.mixin.resource_dir
server = utils.ScriptRunner('127.0.0.1')
try:
server.append('echo "test"')
server.execute()
except ScripRuntimeError:
return
# transfer
self.mixin._copy_resources()
self.mixin._copy_recipes()
# check content of files
for path, content in \
[('resources/res1.txt', 'resource one'),
('resources/resdir/res2.txt', 'resource two'),
('recipes/rec1.pp', 'recipe one'),
('recipes/rec2.pp', 'recipe two')]:
with open(os.path.join(remtmp, path)) as f:
fcont = f.read()
self.assertEqual(fcont, content)
# clean for next test
server = utils.ScriptRunner('127.0.0.1')
server.append('rm -fr %s' % remtmp)
server.execute()
'''
class FakeDroneObserver(DroneObserver):
def __init__(self, *args, **kwargs):
super(FakeDroneObserver, self).__init__(*args, **kwargs)
self.log = []
def applying(self, drone, recipe):
"""
Drone is calling this method when it starts applying recipe.
"""
self.log.append('applying:%s' % recipe)
def checking(self, drone, recipe):
"""
Drone is calling this method when it starts checking if recipe
has been applied.
"""
self.log.append('checking:%s' % recipe)
def finished(self, drone, recipe):
"""
Drone is calling this method when it's finished with recipe
application.
"""
# subclass must implement this method
self.log.append('finished:%s' % recipe)
class FakeDrone(Drone):
def __init__(self, *args, **kwargs):
super(FakeDrone, self).__init__(*args, **kwargs)
self.log = []
def _apply(self, recipe):
self.log.append(recipe)
def _finished(self, recipe):
return True
class DroneTestCase(PackstackTestCaseMixin, TestCase):
def setUp(self):
super(DroneTestCase, self).setUp()
self.observer = FakeDroneObserver()
self.drone = FakeDrone('127.0.0.1')
self.drone.add_recipe('/some/recipe/path1.rec')
self.drone.add_recipe('/some/recipe/path2.rec')
self.drone.add_recipe('/some/recipe/path3a.rec', marker='test')
self.drone.add_recipe('/some/recipe/path3b.rec', marker='test')
self.drone.add_recipe('/some/recipe/path4a.rec', marker='next')
self.drone.add_recipe('/some/recipe/path4b.rec', marker='next')
self.drone.add_recipe('/some/recipe/path5.rec')
def test_drone_behavior(self):
"""
Tests Drone's recipe application order.
"""
self.drone.log = []
self.drone.apply()
rdir = self.drone.recipe_dir
recpaths = [os.path.join(rdir, os.path.basename(i))
for i in self.drone.recipes]
self.assertListEqual(self.drone.log, recpaths)
def test_observer_behavior(self):
"""
Tests DroneObserver calling.
"""
self.drone.set_observer(self.observer)
self.drone.apply()
rdir = self.drone.recipe_dir.rstrip('/')
first = ['applying:/some/recipe/path1.rec',
'checking:%s/path1.rec' % rdir,
'finished:%s/path1.rec' % rdir,
'applying:/some/recipe/path2.rec',
'checking:%s/path2.rec' % rdir,
'finished:%s/path2.rec' % rdir]
last = ['applying:/some/recipe/path5.rec',
'checking:%s/path5.rec' % rdir,
'finished:%s/path5.rec' % rdir]
self.assertListEqual(self.observer.log[:6], first)
self.assertListEqual(self.observer.log[-3:], last)
self.assertEqual(len(self.observer.log), 21)

View File

@@ -1,4 +1,19 @@
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013, Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Test cases for packstack.installer.setup_params module.

View File

@@ -1,4 +1,19 @@
# -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013, Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Test cases for packstack.installer.utils module.