Adjust celery configuration and fix ssh_raw to work with ENV

bootstrap/playbooks/celery.yaml

@@ -15,6 +15,7 @@
     - shell: celery multi stopwait 2  -A solar.orchestration.runner
              chdir={{ celery_dir }}
       tags: [stop]
-    - shell: celery multi start 2 -A solar.orchestration.runner -P:2 gevent -c:1 1 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }}
+    - shell: celery multi start 2 -A solar.orchestration.runner -P -c:2 1000 gevent -c:1 1 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }}
              chdir={{ celery_dir }}
       tags: [master]
+

requirements.txt

@@ -21,6 +21,7 @@ pbr
 pydot
 bunch
 wrapt
+gevent
 # if you want to use riak backend then
 riak
 # if you want to use sql backend then

run.sh

@@ -6,6 +6,6 @@ if [ -d /solar ]; then
 fi
 
 #used only to start celery on docker
-ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags slave,stop
+ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags install
 
 tail -f /var/run/celery/*.log

solar/core/transports/ssh_raw.py

@@ -44,19 +44,24 @@ class RawSSHRunTransport(RunTransport, _RawSSHTransport):
     def run(self, resource, *args, **kwargs):
         log.debug("RAW SSH: %s", args)
 
-        cmds = []
+        commands = []
-        cwd = kwargs.get('cwd')
+        prefix = []
-        if cwd:
-            cmds.append(('cd', cwd))
-
-        cmds.append(args)
-
         if kwargs.get('use_sudo', False):
-            cmds = [('sudo', ) + cmd for cmd in cmds]
+            prefix.append('sudo')
 
-        cmds = [' '.join(cmd) for cmd in cmds]
+        if kwargs.get('cwd'):
+            cmd = prefix + ['cd', kwargs['cwd']]
+            commands.append(' '.join(cmd))
 
-        remote_cmd = '\"%s\"' % ' && '.join(cmds)
+        env = []
+        if 'env' in kwargs:
+            for key, value in kwargs['env'].items():
+                env.append('{}={}'.format(key, value))
+
+        cmd = prefix + env + list(args)
+        commands.append(' '.join(cmd))
+
+        remote_cmd = '\"%s\"' % ' && '.join(commands)
 
         settings = self.settings(resource)
         ssh_cmd = self._ssh_cmd(settings)

templates/nodes_with_transports.yaml

@@ -6,13 +6,23 @@ resources:
     values:
       ssh_user: 'vagrant'
       ssh_key: '/vagrant/.vagrant/machines/solar-dev{{i + 1}}/virtualbox/private_key'
+  - id: rsync{{i}}
+    from: resources/transport_rsync
+    values:
+      user: vagrant
+      key: /vagrant/.vagrant/machines/solar-dev{{i + 1}}/virtualbox/private_key
   - id: transports{{i}}
     from: resources/transports
     values:
-      transports:key: ssh_transport{{i}}::ssh_key
+      transports:
-      transports:user: ssh_transport{{i}}::ssh_user
+        - key: ssh_transport{{i}}::ssh_key
-      transports:port: ssh_transport{{i}}::ssh_port
+          user: ssh_transport{{i}}::ssh_user
-      transports:name: ssh_transport{{i}}::name
+          port: ssh_transport{{i}}::ssh_port
+          name: ssh_transport{{i}}::name
+        - key: rsync{{i}}::key
+          name: rsync{{i}}::name
+          user: rsync{{i}}::user
+          port: rsync{{i}}::port
   - id: node{{i}}
     from: resources/ro_node
     values: