diff --git a/Dockerfile b/Dockerfile index 05bcb3db..41aa3e3a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,9 +7,8 @@ ADD resources /resources ADD templates /templates ADD run.sh /run.sh -RUN apt-get update -RUN apt-get install -y python python-dev python-distribute python-pip \ - libyaml-dev vim libffi-dev libssl-dev git +RUN apt-get upgrade && apt-get update +RUN apt-get install -y python python-dev python-distribute python-pip openssh-client rsync libyaml-dev vim libffi-dev libssl-dev git RUN pip install ansible RUN pip install git+https://github.com/Mirantis/solar.git diff --git a/bootstrap/playbooks/build-main.yaml b/bootstrap/playbooks/build-main.yaml index 6308862e..8f65449b 100644 --- a/bootstrap/playbooks/build-main.yaml +++ b/bootstrap/playbooks/build-main.yaml @@ -3,10 +3,12 @@ - name: Main build script hosts: all sudo: yes + vars: + ssh_ip_mask: "10.0.0.*" tasks: - include: tasks/base.yaml - include: tasks/puppet.yaml - include: tasks/docker.yaml - #- include: celery.yaml tags=['master'] celery_dir=/var/run/celery - include: tasks/cloud_archive.yaml - #- include: tasks/mos.yaml + - include: tasks/ssh_conf.yaml + diff --git a/bootstrap/playbooks/celery.yaml b/bootstrap/playbooks/celery.yaml index e0ad2c33..f02da327 100644 --- a/bootstrap/playbooks/celery.yaml +++ b/bootstrap/playbooks/celery.yaml @@ -18,6 +18,4 @@ - shell: celery multi start 2 -A solar.orchestration.runner -P:2 prefork -c:1 1 -c:2 3 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} chdir={{ celery_dir }} tags: [master] - - shell: celery multi start 1 -A solar.orchestration.runner -Q:1 {{ hostname.stdout }} - chdir={{ celery_dir }} - tags: [slave] + diff --git a/bootstrap/playbooks/files/ssh_conf b/bootstrap/playbooks/files/ssh_conf new file mode 100644 index 00000000..da7b2483 --- /dev/null +++ b/bootstrap/playbooks/files/ssh_conf @@ -0,0 +1,2 @@ +Host {{ssh_ip_mask}} + StrictHostKeyChecking no diff --git a/bootstrap/playbooks/solar.yaml b/bootstrap/playbooks/solar.yaml index 519c44f5..e2211caf 100644 --- a/bootstrap/playbooks/solar.yaml +++ b/bootstrap/playbooks/solar.yaml @@ -2,12 +2,15 @@ - hosts: all sudo: yes + vars: + ssh_ip_mask: "10.0.0.*" tasks: # upgrade pbr first, old version throws strange errors - shell: pip install pbr -U # Setup development env for solar - shell: pip install -e . chdir=/vagrant - shell: pip install git+git://github.com/Mirantis/solar-agent.git + - include: tasks/ssh_conf.yaml - hosts: all tasks: diff --git a/bootstrap/playbooks/tasks/ssh_conf.yaml b/bootstrap/playbooks/tasks/ssh_conf.yaml new file mode 100644 index 00000000..1c99ff94 --- /dev/null +++ b/bootstrap/playbooks/tasks/ssh_conf.yaml @@ -0,0 +1,3 @@ +--- + +- template: src=files/ssh_conf dest=/root/.ssh/config diff --git a/doc/source/transports.rst b/doc/source/transports.rst index 950a681e..f5ead257 100644 --- a/doc/source/transports.rst +++ b/doc/source/transports.rst @@ -31,6 +31,16 @@ Currently there are following sync transports available: * solar_agent * torrent +Ssh host key checking +--------------------- +Solar wont disable strict host key checking by default, so before working +with solar ensure that strict host key checking is disabled, or all target hosts added to .ssh/known_hosts file. + +Example of .ssh/config :: + + Host 10.0.0.* + StrictHostKeyChecking no + Run transport ------------- diff --git a/docker-compose.yml b/docker-compose.yml index 0fd699d8..fd60aade 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,8 @@ solar-celery: - /vagrant/templates:/vagrant/templates - /vagrant/resources:/vagrant/resources - /vagrant/library:/vagrant/library + - ~/.ssh:/root/.ssh + - ./bootstrap/playbooks/celery.yaml:/celery.yaml environment: - REDIS_HOST=redis - REDIS_PORT=6379 diff --git a/examples/hosts_file/hosts.py b/examples/hosts_file/hosts.py index 54d04583..19342ced 100644 --- a/examples/hosts_file/hosts.py +++ b/examples/hosts_file/hosts.py @@ -10,12 +10,12 @@ from solar.dblayer.model import ModelMeta def run(): ModelMeta.remove_all() - resources = vr.create('nodes', 'templates/nodes_with_transports.yaml', {'count': 2}) - nodes = [x for x in resources if x.name.startswith('node')] - node1, node2 = nodes + resources = vr.create('nodes', 'templates/nodes.yaml', {'count': 2}) + + node1, node2 = [x for x in resources if x.name.startswith('node')] + hosts1, hosts2 = [x for x in resources + if x.name.startswith('hosts_file')] - hosts1 = vr.create('hosts_file1', 'resources/hosts_file', {})[0] - hosts2 = vr.create('hosts_file2', 'resources/hosts_file', {})[0] node1.connect(hosts1, { 'name': 'hosts:name', 'ip': 'hosts:ip', @@ -36,5 +36,4 @@ def run(): 'ip': 'hosts:ip', }) - run() diff --git a/resources/transport_rsync/meta.yaml b/resources/transport_rsync/meta.yaml new file mode 100644 index 00000000..5cd3a417 --- /dev/null +++ b/resources/transport_rsync/meta.yaml @@ -0,0 +1,23 @@ +id: transport_rsync +input: + key: + schema: str! + value: + user: + schema: str! + value: + name: + schema: str! + value: rsync + location_id: + schema: str + value: + reverse: True + is_own: False + transports_id: + schema: str + value: + is_emit: False + port: + schema: int + value: 3579 diff --git a/run.sh b/run.sh index 1d1f3cf7..6894d773 100755 --- a/run.sh +++ b/run.sh @@ -6,6 +6,6 @@ if [ -d /solar ]; then fi #used only to start celery on docker -ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags slave,stop +ansible-playbook -v -i "localhost," -c local /celery.yaml --skip-tags install tail -f /var/run/celery/*.log diff --git a/solar/core/handlers/ansible_template.py b/solar/core/handlers/ansible_template.py index 02d2c8e2..85170a3a 100644 --- a/solar/core/handlers/ansible_template.py +++ b/solar/core/handlers/ansible_template.py @@ -19,7 +19,6 @@ import os from solar.core.handlers.base import SOLAR_TEMP_LOCAL_LOCATION from solar.core.handlers.base import TempFileHandler from solar.core.log import log -from solar import errors # otherwise fabric will sys.exit(1) in case of errors env.warn_only = True @@ -52,15 +51,8 @@ class AnsibleTemplate(TempFileHandler): '-i', remote_inventory_file, remote_playbook_file] log.debug('EXECUTING: %s', ' '.join(call_args)) - out = self.transport_run.run(resource, *call_args) - log.debug(out) - if out.failed: - raise errors.SolarError(out) - - # with fabric_api.shell_env(ANSIBLE_HOST_KEY_CHECKING='False'): - # out = fabric_api.local(' '.join(call_args), capture=True) - # if out.failed: - # raise errors.SolarError(out) + rst = self.transport_run.run(resource, *call_args) + self.verify_run_result(call_args, rst) def _create_inventory(self, r): directory = self.dirs[r.name] diff --git a/solar/core/handlers/base.py b/solar/core/handlers/base.py index 36e5086b..e90818ed 100644 --- a/solar/core/handlers/base.py +++ b/solar/core/handlers/base.py @@ -21,6 +21,7 @@ import tempfile from solar.core.log import log from solar.core.transports.ssh import SSHRunTransport from solar.core.transports.ssh import SSHSyncTransport +from solar import errors from solar import utils @@ -42,6 +43,13 @@ class BaseHandler(object): self.transport_sync.bind_with(self.transport_run) self.transport_run.bind_with(self.transport_sync) + def verify_run_result(self, cmd, result): + rc, out, err = result + log.debug('CMD %r RC %s OUT %s ERR %s', cmd, rc, out, err) + if rc: + message = 'CMD %r failed RC %s ERR %s' % (cmd, rc, err) + raise errors.SolarError(message) + def __enter__(self): return self diff --git a/solar/core/handlers/puppet.py b/solar/core/handlers/puppet.py index c52f88b5..524130c9 100644 --- a/solar/core/handlers/puppet.py +++ b/solar/core/handlers/puppet.py @@ -46,7 +46,7 @@ class Puppet(TempFileHandler): cmd_args.append('--modulepath={}'.format( resource.args['puppet_modules'])) - cmd = self.transport_run.run( + rc, out, err = self.transport_run.run( resource, *cmd_args, env={ @@ -55,12 +55,12 @@ class Puppet(TempFileHandler): use_sudo=True, warn_only=True ) + log.debug('CMD %r RC %s OUT %s ERR %s', cmd_args, rc, out, err) # 0 - no changes, 2 - successfull changes - if cmd.return_code not in [0, 2]: + if rc not in [0, 2]: raise errors.SolarError( - 'Puppet for {} failed with {}'.format( - resource.name, cmd.return_code)) - return cmd + 'Puppet for {} failed with RC {}'.format( + resource.name, rc)) def _make_args(self, resource): return {resource.name: {'input': resource.args}} diff --git a/solar/core/handlers/shell.py b/solar/core/handlers/shell.py index da4875a6..a028d3f0 100644 --- a/solar/core/handlers/shell.py +++ b/solar/core/handlers/shell.py @@ -18,7 +18,6 @@ import os from solar.core.handlers.base import SOLAR_TEMP_LOCAL_LOCATION from solar.core.handlers.base import TempFileHandler from solar.core.log import log -from solar import errors class Shell(TempFileHandler): @@ -36,15 +35,10 @@ class Shell(TempFileHandler): self.transport_sync.copy(resource, self.dst, '/tmp') self.transport_sync.sync_all() - cmd = self.transport_run.run( + rst = self.transport_run.run( resource, 'bash', action_file_name, use_sudo=True, warn_only=True ) - - if cmd.return_code: - raise errors.SolarError( - 'Bash execution for {} failed with {}'.format( - resource.name, cmd.return_code)) - return cmd + self.verify_run_results(['bash', action_file_name], rst) diff --git a/solar/core/resource/virtual_resource.py b/solar/core/resource/virtual_resource.py index eb898680..f9863bbe 100644 --- a/solar/core/resource/virtual_resource.py +++ b/solar/core/resource/virtual_resource.py @@ -266,14 +266,17 @@ def parse_list_input(r_input, args): connections = [] assignments = {} for arg in args: - if is_connection(arg): + if isinstance(arg, dict): + n_connections, n_assign = parse_dict_input( + r_input, arg) + connections.extend(n_connections) + if n_assign: + add_assignment(assignments, r_input, n_assign) + elif is_connection(arg): c = parse_connection(r_input, arg) connections.append(c) else: - try: - assignments[r_input].append(arg) - except KeyError: - assignments[r_input] = [arg] + add_assignment(assignments, r_input, arg) return connections, assignments @@ -293,6 +296,13 @@ def parse_dict_input(r_input, args): return connections, assignments +def add_assignment(assignments, r_input, arg): + try: + assignments[r_input].append(arg) + except KeyError: + assignments[r_input] = [arg] + + def is_connection(arg): if isinstance(arg, basestring) and '::' in arg: return True diff --git a/solar/core/transports/base.py b/solar/core/transports/base.py index 168578b5..1dbca863 100644 --- a/solar/core/transports/base.py +++ b/solar/core/transports/base.py @@ -12,6 +12,9 @@ # License for the specific language governing permissions and limitations # under the License. +from solar.core.log import log +from solar import errors + class Executor(object): @@ -38,7 +41,13 @@ class Executor(object): def run(self, transport): if self.valid: - self._executor(transport) + result = self._executor(transport) + if isinstance(result, tuple) and len(result) == 3: + # TODO Include file information in result + rc, out, err = result + log.debug('RC %s OUT %s ERR %s', rc, out, err) + if rc: + raise errors.SolarError(err) class SolarRunResultWrp(object): diff --git a/solar/core/transports/bat.py b/solar/core/transports/bat.py index 3c7442a6..5843ed88 100644 --- a/solar/core/transports/bat.py +++ b/solar/core/transports/bat.py @@ -17,8 +17,8 @@ from solar.core.transports.base import RunTransport from solar.core.transports.base import SolarTransport from solar.core.transports.base import SyncTransport from solar.core.transports.rsync import RsyncSyncTransport -from solar.core.transports.ssh import SSHRunTransport from solar.core.transports.ssh import SSHSyncTransport +from solar.core.transports.ssh_raw import RawSSHRunTransport try: from solar.core.transports.solar_agent_transport import SolarAgentRunTransport # NOQA from solar.core.transports.solar_agent_transport import SolarAgentSyncTransport # NOQA @@ -42,7 +42,7 @@ KNOWN_SYNC_TRANSPORTS = { KNOWN_RUN_TRANSPORTS = { - 'ssh': SSHRunTransport + 'ssh': RawSSHRunTransport } diff --git a/solar/core/transports/rsync.py b/solar/core/transports/rsync.py index 88d27b93..da03722c 100644 --- a/solar/core/transports/rsync.py +++ b/solar/core/transports/rsync.py @@ -12,11 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -from fabric import api as fabric_api - from solar.core.log import log from solar.core.transports.base import Executor from solar.core.transports.base import SyncTransport +from solar.utils import execute # XXX: # currently we don't support key verification or acceptation @@ -55,9 +54,8 @@ class RsyncSyncTransport(SyncTransport): _from=_from, _to=_to) - rsync_executor = lambda transport: fabric_api.local( - rsync_cmd - ) + rsync_executor = lambda transport: execute( + rsync_cmd, shell=True) log.debug("RSYNC CMD: %r" % rsync_cmd) diff --git a/solar/core/transports/ssh_raw.py b/solar/core/transports/ssh_raw.py index 33a61aec..31a66a7e 100644 --- a/solar/core/transports/ssh_raw.py +++ b/solar/core/transports/ssh_raw.py @@ -12,27 +12,31 @@ # License for the specific language governing permissions and limitations # under the License. -from fabric import api as fabric_api from solar.core.log import log from solar.core.transports.base import RunTransport +from solar.utils import execute class _RawSSHTransport(object): - def _ssh_props(self, resource): - return { - 'ssh_key': resource.args['ssh_key'].value, - 'ssh_user': resource.args['ssh_user'].value - } + def settings(self, resource): + transport = self.get_transport_data(resource) + host = resource.ip() + user = transport['user'] + port = transport['port'] + key = transport['key'] + return {'ssh_user': user, + 'ssh_key': key, + 'port': port, + 'ip': host} - def _ssh_command_host(self, resource): - return '{}@{}'.format(resource.args['ssh_user'].value, - resource.args['ip'].value) + def _ssh_command_host(self, settings): + return '{}@{}'.format(settings['ssh_user'], + settings['ip']) - def _ssh_cmd(self, resource): - props = self._ssh_props(resource) - return ('ssh', '-i', props['ssh_key']) + def _ssh_cmd(self, settings): + return ('ssh', '-i', settings['ssh_key']) class RawSSHRunTransport(RunTransport, _RawSSHTransport): @@ -40,23 +44,30 @@ class RawSSHRunTransport(RunTransport, _RawSSHTransport): def run(self, resource, *args, **kwargs): log.debug("RAW SSH: %s", args) - cmds = [] - cwd = kwargs.get('cwd') - if cwd: - cmds.append(('cd', cwd)) - - cmds.append(args) - + commands = [] + prefix = [] if kwargs.get('use_sudo', False): - cmds = [('sudo', ) + cmd for cmd in cmds] + prefix.append('sudo') - cmds = [' '.join(cmd) for cmd in cmds] + if kwargs.get('cwd'): + cmd = prefix + ['cd', kwargs['cwd']] + commands.append(' '.join(cmd)) - remote_cmd = '\"%s\"' % ' && '.join(cmds) + env = [] + if 'env' in kwargs: + for key, value in kwargs['env'].items(): + env.append('{}={}'.format(key, value)) - ssh_cmd = self._ssh_cmd(resource) - ssh_cmd += (self._ssh_command_host(resource), remote_cmd) + cmd = prefix + env + list(args) + commands.append(' '.join(cmd)) - log.debug("SSH CMD: %r", ssh_cmd) + remote_cmd = '\"%s\"' % ' && '.join(commands) - return fabric_api.local(' '.join(ssh_cmd)) + settings = self.settings(resource) + ssh_cmd = self._ssh_cmd(settings) + ssh_cmd += (self._ssh_command_host(settings), remote_cmd) + + log.debug("RAW SSH CMD: %r", ssh_cmd) + # TODO convert it to SolarRunResult + + return execute(' '.join(ssh_cmd), shell=True) diff --git a/solar/test/test_virtual_resource.py b/solar/test/test_virtual_resource.py index 997040b6..ec870095 100644 --- a/solar/test/test_virtual_resource.py +++ b/solar/test/test_virtual_resource.py @@ -202,6 +202,19 @@ def test_parse_connection_disable_events(): assert correct_connection == connection +def test_parse_list_of_connected_dicts(): + inputs = {'list': [ + {'key': 'emitter1::key'}, + {'key': 'emitter2::key'}]} + connections, assignments = vr.parse_inputs(inputs) + assert assignments == {} + assert connections == [ + {'child_input': 'list:key', 'parent_input': 'key', + 'parent': 'emitter1', 'events': None}, + {'child_input': 'list:key', 'parent_input': 'key', + 'parent': 'emitter2', 'events': None}] + + def test_setting_location(tmpdir): # XXX: make helper for it base_path = os.path.join( diff --git a/solar/utils.py b/solar/utils.py index 813b8488..5f30214a 100644 --- a/solar/utils.py +++ b/solar/utils.py @@ -43,6 +43,17 @@ def communicate(command, data): stderr=subprocess.PIPE) return popen.communicate(input=data)[0] + +def execute(command, shell=False): + popen = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=shell) + out, err = popen.communicate() + return popen.returncode, out, err + + # Configure jinja2 filters jinja_env_with_filters = Environment() jinja_env_with_filters.filters['to_json'] = to_json diff --git a/templates/nodes.yaml b/templates/nodes.yaml index 3c2e75a9..5129ab49 100644 --- a/templates/nodes.yaml +++ b/templates/nodes.yaml @@ -7,13 +7,23 @@ resources: values: ssh_user: 'vagrant' ssh_key: '/vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key' + - id: rsync{{j}} + from: resources/transport_rsync + values: + user: vagrant + key: /vagrant/.vagrant/machines/solar-dev{{j}}/virtualbox/private_key - id: transports{{j}} from: resources/transports values: - transports:key: ssh_transport{{j}}::ssh_key - transports:user: ssh_transport{{j}}::ssh_user - transports:port: ssh_transport{{j}}::ssh_port - transports:name: ssh_transport{{j}}::name + transports: + - key: ssh_transport{{j}}::ssh_key + user: ssh_transport{{j}}::ssh_user + port: ssh_transport{{j}}::ssh_port + name: ssh_transport{{j}}::name + - key: rsync{{j}}::key + name: rsync{{j}}::name + user: rsync{{j}}::user + port: rsync{{j}}::port - id: node{{j}} from: resources/ro_node values: diff --git a/templates/nodes_with_transports.yaml b/templates/nodes_with_transports.yaml index f0501da4..a185b195 100644 --- a/templates/nodes_with_transports.yaml +++ b/templates/nodes_with_transports.yaml @@ -6,13 +6,23 @@ resources: values: ssh_user: 'vagrant' ssh_key: '/vagrant/.vagrant/machines/solar-dev{{i + 1}}/virtualbox/private_key' + - id: rsync{{i}} + from: resources/transport_rsync + values: + user: vagrant + key: /vagrant/.vagrant/machines/solar-dev{{i + 1}}/virtualbox/private_key - id: transports{{i}} from: resources/transports values: - transports:key: ssh_transport{{i}}::ssh_key - transports:user: ssh_transport{{i}}::ssh_user - transports:port: ssh_transport{{i}}::ssh_port - transports:name: ssh_transport{{i}}::name + transports: + - key: ssh_transport{{i}}::ssh_key + user: ssh_transport{{i}}::ssh_user + port: ssh_transport{{i}}::ssh_port + name: ssh_transport{{i}}::name + - key: rsync{{i}}::key + name: rsync{{i}}::name + user: rsync{{i}}::user + port: rsync{{i}}::port - id: node{{i}} from: resources/ro_node values: