Some fixes in shotgun

shotgun/bin/example.json

@@ -33,6 +33,24 @@
                     "password": "PASSWORD"
                }
             }
+        ],
+        "slave": [
+            {
+                "type": "command",
+                "command": "ps auxww"
+            },
+            {
+                "type": "command",
+                "command": "top"
+            },
+            {
+                "type": "command",
+                "command": "ip a"
+            },
+            {
+                "type": "file",
+                "path": "/etc/naily.facts"
+            }
         ]
     }
 }

shotgun/shotgun/config.py

@@ -2,6 +2,7 @@ import time
 
 from shotgun import settings
 
+
 class Config(object):
     def __init__(self, data=None):
         self.data = data
@@ -18,6 +19,10 @@ class Config(object):
             target = self._timestamp(target)
         return target
 
+    @property
+    def lastdump(self):
+        return self.data.get("lastdump", settings.LASTDUMP)
+
     @property
     def objects(self):
         for role, hosts in self.data["dump_roles"].iteritems():

shotgun/shotgun/driver.py

@@ -1,18 +1,18 @@
 import os
 import re
-import logging
 import tempfile
 
 import fabric.api
 
+from shotgun.logger import logger
 from shotgun.utils import is_local
 from shotgun.utils import execute
 
-logger = logging.getLogger()
 
 class CommandOut(object):
     pass
 
+
 class Driver(object):
     @classmethod
     def getDriver(cls, data, conf):
@@ -22,10 +22,12 @@ class Driver(object):
             "dir": Dir,
             "subs": Subs,
             "postgres": Postgres,
+            "command": Command,
         }.get(driver_type, cls)(data, conf)
 
     def __init__(self, data, conf):
-        logger.debug("Initializing driver %s", self.__class__.__name__)
+        logger.debug("Initializing driver %s: host=%s",
+            self.__class__.__name__, data.get("host"))
         self.data = data
         self.host = self.data.get("host", "localhost")
         self.local = is_local(self.host)
@@ -35,14 +37,20 @@ class Driver(object):
         raise NotImplementedError
 
     def command(self, command):
+        """
+        This method is able to run only simple commands not series of them
+        cmd1 | cmd2 | cmd3 does not work in general. It works only for localhost
+        because locally command is launched with 'execute' util method.
+        """
         out = CommandOut()
         if not self.local:
             with fabric.api.settings(host_string=self.host):
                 logger.debug("Running remote command: "
                              "host: %s command: %s", self.host, command)
-                out.stdout = fabric.api.run(command, pty=True)
-                out.return_code = result.return_code
-                out.stderr = result.stderr
+                output = fabric.api.run(command, pty=True)
+                out.stdout = output
+                out.return_code = output.return_code
+                out.stderr = output.stderr
         else:
             logger.debug("Running local command: %s", command)
             out.return_code, out.stdout, out.stderr = execute(command)
@@ -63,8 +71,10 @@ class File(Driver):
     def __init__(self, data, conf):
         super(File, self).__init__(data, conf)
         self.path = self.data["path"]
+        logger.debug("File to get: %s", self.path)
         self.target_path = os.path.join(
-            self.conf.target, self.host, os.path.relpath(self.path, "/"))
+            self.conf.target, self.host, self.path.lstrip("/"))
+        logger.debug("File to save: %s", self.target_path)
 
     def snapshot(self):
         self.get(self.path, self.target_path)
@@ -132,7 +142,6 @@ class Subs(File):
         tf.close()
 
 
-
 class Postgres(Driver):
     def __init__(self, data, conf):
         super(Postgres, self).__init__(data, conf)
@@ -154,4 +163,22 @@ class Postgres(Driver):
         self.command("rm -f %s" % temp)
 
 
+class Command(Driver):
+    def __init__(self, data, conf):
+        super(Command, self).__init__(data, conf)
+        self.cmdname = self.data["command"]
+        self.to_file = self.data["to_file"]
+        self.target_path = os.path.join(
+            self.conf.target, self.host, "commands", self.to_file)
+
+    def snapshot(self):
+        out = self.command(self.cmdname)
+        execute("mkdir -p {0}".format(os.path.dirname(self.target_path)))
+        with open(self.target_path, "w") as f:
+            f.write("===== COMMAND =====: {0}\n".format(self.cmdname))
+            f.write("===== RETURN CODE =====: {0}\n".format(out.return_code))
+            f.write("===== STDOUT =====:\n")
+            f.write(out.stdout)
+            f.write("\n===== STDERR =====:\n")
+            f.write(out.stderr)
 

shotgun/shotgun/logger.py

@@ -0,0 +1,4 @@
+import logging
+
+
+logger = logging.getLogger("nailgun.shotgun")

shotgun/shotgun/manager.py

@@ -1,17 +1,20 @@
 import os
 import logging
 
+from shotgun.logger import logger
 from shotgun.driver import Driver
 from shotgun.utils import execute
 
-logger = logging.getLogger()
 
 class Manager(object):
     def __init__(self, conf):
+        logger.debug("Initializing snapshot manager")
         self.conf = conf
 
     def snapshot(self):
+        logger.debug("Making snapshot")
         for obj_data in self.conf.objects:
+            logger.debug("Dumping: %s", obj_data)
             driver = Driver.getDriver(obj_data, self.conf)
             driver.snapshot()
         logger.debug("Archiving dump directory: %s", self.conf.target)
@@ -19,6 +22,9 @@ class Manager(object):
                 "".format(self.conf.target,
                           os.path.dirname(self.conf.target),
                           os.path.basename(self.conf.target)))
+        execute("rm -r {0}".format(self.conf.target))
+        with open(self.conf.lastdump, "w") as fo:
+            fo.write("%s.tgz" % self.conf.target)
         return "%s.tgz" % self.conf.target
 
 

shotgun/shotgun/settings.py

@@ -1,2 +1,3 @@
 TARGET = "/tmp/snapshot"
+LASTDUMP = "/tmp/snapshot_last"
 TIMESTAMP = True

shotgun/shotgun/utils.py

@@ -3,10 +3,8 @@ import re
 import os
 import shlex
 import subprocess
-import logging
 
-
-logger = logging.getLogger()
+from shotgun.logger import logger
 
 
 def hostname():
@@ -28,7 +26,13 @@ def is_local(name):
         return True
     return False
 
+
 def execute(command, to_filename=None):
+    """
+    This method is used for running shell commands locally
+    and it is able to run series of commands with pipes
+    cmd1 | cmd2 | cmd3
+    """
     logger.debug("Trying to execute command: %s", command)
     commands = [c.strip() for c in re.split(ur'\|', command)]
     env = os.environ
@@ -40,15 +44,19 @@ def execute(command, to_filename=None):
 
     process = []
     for c in commands:
-        process.append(subprocess.Popen(
-            shlex.split(c),
-            env=env,
-            stdin=(process[-1].stdout if process else None),
-            stdout=(to_file
-                    if (len(process) == len(commands) - 1) and to_file
-                    else subprocess.PIPE),
-            stderr=(subprocess.PIPE)
-        ))
+        try:
+            process.append(subprocess.Popen(
+                shlex.split(c),
+                env=env,
+                stdin=(process[-1].stdout if process else None),
+                stdout=(to_file
+                        if (len(process) == len(commands) - 1) and to_file
+                        else subprocess.PIPE),
+                stderr=(subprocess.PIPE)
+            ))
+        except OSError as e:
+            return (1, "", "%s\n" % str(e))
+
         if len(process) >= 2:
             process[-2].stdout.close()
     stdout, stderr = process[-1].communicate()