From 0b12c5afc074395f3a127ee43a5b14b65f9cf019 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Mon, 9 Sep 2013 19:08:49 +0400 Subject: [PATCH] Some fixes in shotgun --- shotgun/bin/{snapshot.json => example.json} | 18 +++++++++ shotgun/bin/{snapshot.py => example.py} | 0 shotgun/{bin => draft}/testing.py | 0 shotgun/shotgun/config.py | 5 +++ shotgun/shotgun/driver.py | 43 +++++++++++++++++---- shotgun/shotgun/logger.py | 4 ++ shotgun/shotgun/manager.py | 8 +++- shotgun/shotgun/settings.py | 1 + shotgun/shotgun/utils.py | 32 +++++++++------ 9 files changed, 90 insertions(+), 21 deletions(-) rename shotgun/bin/{snapshot.json => example.json} (67%) rename shotgun/bin/{snapshot.py => example.py} (100%) rename shotgun/{bin => draft}/testing.py (100%) create mode 100644 shotgun/shotgun/logger.py diff --git a/shotgun/bin/snapshot.json b/shotgun/bin/example.json similarity index 67% rename from shotgun/bin/snapshot.json rename to shotgun/bin/example.json index d4aa0cdcdd..40c3197757 100644 --- a/shotgun/bin/snapshot.json +++ b/shotgun/bin/example.json @@ -33,6 +33,24 @@ "password": "PASSWORD" } } + ], + "slave": [ + { + "type": "command", + "command": "ps auxww" + }, + { + "type": "command", + "command": "top" + }, + { + "type": "command", + "command": "ip a" + }, + { + "type": "file", + "path": "/etc/naily.facts" + } ] } } \ No newline at end of file diff --git a/shotgun/bin/snapshot.py b/shotgun/bin/example.py similarity index 100% rename from shotgun/bin/snapshot.py rename to shotgun/bin/example.py diff --git a/shotgun/bin/testing.py b/shotgun/draft/testing.py similarity index 100% rename from shotgun/bin/testing.py rename to shotgun/draft/testing.py diff --git a/shotgun/shotgun/config.py b/shotgun/shotgun/config.py index d20ae2d43e..263d1c0802 100644 --- a/shotgun/shotgun/config.py +++ b/shotgun/shotgun/config.py @@ -2,6 +2,7 @@ import time from shotgun import settings + class Config(object): def __init__(self, data=None): self.data = data @@ -18,6 +19,10 @@ class Config(object): target = self._timestamp(target) return target + @property + def lastdump(self): + return self.data.get("lastdump", settings.LASTDUMP) + @property def objects(self): for role, hosts in self.data["dump_roles"].iteritems(): diff --git a/shotgun/shotgun/driver.py b/shotgun/shotgun/driver.py index afd0f6e4c3..d7ce1baf27 100644 --- a/shotgun/shotgun/driver.py +++ b/shotgun/shotgun/driver.py @@ -1,18 +1,18 @@ import os import re -import logging import tempfile import fabric.api +from shotgun.logger import logger from shotgun.utils import is_local from shotgun.utils import execute -logger = logging.getLogger() class CommandOut(object): pass + class Driver(object): @classmethod def getDriver(cls, data, conf): @@ -22,10 +22,12 @@ class Driver(object): "dir": Dir, "subs": Subs, "postgres": Postgres, + "command": Command, }.get(driver_type, cls)(data, conf) def __init__(self, data, conf): - logger.debug("Initializing driver %s", self.__class__.__name__) + logger.debug("Initializing driver %s: host=%s", + self.__class__.__name__, data.get("host")) self.data = data self.host = self.data.get("host", "localhost") self.local = is_local(self.host) @@ -35,14 +37,20 @@ class Driver(object): raise NotImplementedError def command(self, command): + """ + This method is able to run only simple commands not series of them + cmd1 | cmd2 | cmd3 does not work in general. It works only for localhost + because locally command is launched with 'execute' util method. + """ out = CommandOut() if not self.local: with fabric.api.settings(host_string=self.host): logger.debug("Running remote command: " "host: %s command: %s", self.host, command) - out.stdout = fabric.api.run(command, pty=True) - out.return_code = result.return_code - out.stderr = result.stderr + output = fabric.api.run(command, pty=True) + out.stdout = output + out.return_code = output.return_code + out.stderr = output.stderr else: logger.debug("Running local command: %s", command) out.return_code, out.stdout, out.stderr = execute(command) @@ -63,8 +71,10 @@ class File(Driver): def __init__(self, data, conf): super(File, self).__init__(data, conf) self.path = self.data["path"] + logger.debug("File to get: %s", self.path) self.target_path = os.path.join( - self.conf.target, self.host, os.path.relpath(self.path, "/")) + self.conf.target, self.host, self.path.lstrip("/")) + logger.debug("File to save: %s", self.target_path) def snapshot(self): self.get(self.path, self.target_path) @@ -132,7 +142,6 @@ class Subs(File): tf.close() - class Postgres(Driver): def __init__(self, data, conf): super(Postgres, self).__init__(data, conf) @@ -154,4 +163,22 @@ class Postgres(Driver): self.command("rm -f %s" % temp) +class Command(Driver): + def __init__(self, data, conf): + super(Command, self).__init__(data, conf) + self.cmdname = self.data["command"] + self.to_file = self.data["to_file"] + self.target_path = os.path.join( + self.conf.target, self.host, "commands", self.to_file) + + def snapshot(self): + out = self.command(self.cmdname) + execute("mkdir -p {0}".format(os.path.dirname(self.target_path))) + with open(self.target_path, "w") as f: + f.write("===== COMMAND =====: {0}\n".format(self.cmdname)) + f.write("===== RETURN CODE =====: {0}\n".format(out.return_code)) + f.write("===== STDOUT =====:\n") + f.write(out.stdout) + f.write("\n===== STDERR =====:\n") + f.write(out.stderr) diff --git a/shotgun/shotgun/logger.py b/shotgun/shotgun/logger.py new file mode 100644 index 0000000000..8a567e8978 --- /dev/null +++ b/shotgun/shotgun/logger.py @@ -0,0 +1,4 @@ +import logging + + +logger = logging.getLogger("nailgun.shotgun") diff --git a/shotgun/shotgun/manager.py b/shotgun/shotgun/manager.py index ccc43909ab..e0cfd13c99 100644 --- a/shotgun/shotgun/manager.py +++ b/shotgun/shotgun/manager.py @@ -1,17 +1,20 @@ import os import logging +from shotgun.logger import logger from shotgun.driver import Driver from shotgun.utils import execute -logger = logging.getLogger() class Manager(object): def __init__(self, conf): + logger.debug("Initializing snapshot manager") self.conf = conf def snapshot(self): + logger.debug("Making snapshot") for obj_data in self.conf.objects: + logger.debug("Dumping: %s", obj_data) driver = Driver.getDriver(obj_data, self.conf) driver.snapshot() logger.debug("Archiving dump directory: %s", self.conf.target) @@ -19,6 +22,9 @@ class Manager(object): "".format(self.conf.target, os.path.dirname(self.conf.target), os.path.basename(self.conf.target))) + execute("rm -r {0}".format(self.conf.target)) + with open(self.conf.lastdump, "w") as fo: + fo.write("%s.tgz" % self.conf.target) return "%s.tgz" % self.conf.target diff --git a/shotgun/shotgun/settings.py b/shotgun/shotgun/settings.py index af4c258b94..954cc54c95 100644 --- a/shotgun/shotgun/settings.py +++ b/shotgun/shotgun/settings.py @@ -1,2 +1,3 @@ TARGET = "/tmp/snapshot" +LASTDUMP = "/tmp/snapshot_last" TIMESTAMP = True diff --git a/shotgun/shotgun/utils.py b/shotgun/shotgun/utils.py index ced7f3069e..21bebf93f6 100644 --- a/shotgun/shotgun/utils.py +++ b/shotgun/shotgun/utils.py @@ -3,10 +3,8 @@ import re import os import shlex import subprocess -import logging - -logger = logging.getLogger() +from shotgun.logger import logger def hostname(): @@ -28,7 +26,13 @@ def is_local(name): return True return False + def execute(command, to_filename=None): + """ + This method is used for running shell commands locally + and it is able to run series of commands with pipes + cmd1 | cmd2 | cmd3 + """ logger.debug("Trying to execute command: %s", command) commands = [c.strip() for c in re.split(ur'\|', command)] env = os.environ @@ -40,15 +44,19 @@ def execute(command, to_filename=None): process = [] for c in commands: - process.append(subprocess.Popen( - shlex.split(c), - env=env, - stdin=(process[-1].stdout if process else None), - stdout=(to_file - if (len(process) == len(commands) - 1) and to_file - else subprocess.PIPE), - stderr=(subprocess.PIPE) - )) + try: + process.append(subprocess.Popen( + shlex.split(c), + env=env, + stdin=(process[-1].stdout if process else None), + stdout=(to_file + if (len(process) == len(commands) - 1) and to_file + else subprocess.PIPE), + stderr=(subprocess.PIPE) + )) + except OSError as e: + return (1, "", "%s\n" % str(e)) + if len(process) >= 2: process[-2].stdout.close() stdout, stderr = process[-1].communicate()