From 6078670c8a52fa1375561c8b98ea4013a4bcba74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Ole=C5=9B?= Date: Thu, 26 Nov 2015 12:29:42 +0100 Subject: [PATCH 1/5] Run autopep8 --- .travis.yml | 2 +- solar/cli/base.py | 1 + solar/cli/events.py | 2 +- solar/cli/executors.py | 24 +++++- solar/cli/main.py | 27 +++---- solar/cli/orch.py | 17 ++-- solar/cli/resource.py | 31 ++++++-- solar/cli/system_log.py | 11 ++- solar/cli/uids_history.py | 5 +- solar/config.py | 19 ++++- solar/core/actions.py | 2 +- solar/core/handlers/__init__.py | 1 + solar/core/handlers/ansible_playbook.py | 6 +- solar/core/handlers/ansible_template.py | 12 ++- solar/core/handlers/base.py | 5 +- solar/core/handlers/puppet.py | 4 +- solar/core/handlers/python.py | 1 + solar/core/handlers/shell.py | 4 +- solar/core/log.py | 6 +- solar/core/provider.py | 9 ++- solar/core/resource/resource.py | 14 ++-- solar/core/resource/virtual_resource.py | 13 +-- solar/core/signals.py | 6 +- solar/core/tags_set_parser.py | 13 +-- solar/core/transports/base.py | 3 +- solar/core/transports/bat.py | 3 +- .../core/transports/helpers/solar_torrent.py | 16 ++-- .../core/transports/solar_agent_transport.py | 5 +- solar/core/transports/ssh_raw.py | 2 - solar/core/transports/torrent.py | 10 ++- solar/core/validation.py | 13 +-- solar/events/api.py | 5 +- solar/events/controls.py | 7 +- solar/orchestration/consts.py | 3 - solar/orchestration/filters.py | 7 +- solar/orchestration/graph.py | 3 +- solar/orchestration/limits.py | 9 ++- solar/orchestration/runner.py | 4 +- solar/orchestration/tasks.py | 8 +- solar/orchestration/utils.py | 3 +- solar/system_log/change.py | 37 +++++---- solar/system_log/consts.py | 6 +- solar/system_log/data.py | 8 +- solar/template.py | 9 ++- solar/test/conftest.py | 17 ++-- solar/test/test_diff_generation.py | 17 ++-- solar/test/test_events.py | 15 ++-- solar/test/test_graph_api.py | 10 ++- solar/test/test_graph_filtering.py | 17 ++-- solar/test/test_limits.py | 4 +- solar/test/test_operations_with_tags.py | 6 +- solar/test/test_resource.py | 1 + solar/test/test_signals.py | 43 ++++++---- solar/test/test_system_log_api.py | 79 ++++++++++--------- solar/test/test_system_log_details.py | 9 ++- solar/test/test_traversal.py | 3 + solar/test/test_validation.py | 1 + solar/test/test_virtual_resource.py | 47 ++++++----- solar/utils.py | 14 ++-- tox.ini | 7 +- 60 files changed, 419 insertions(+), 267 deletions(-) diff --git a/.travis.yml b/.travis.yml index 570cc00b..9951f676 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ install: - pip-accel install coveralls - pip-accel install -r test-requirements.txt script: - - py.test --cov=solar -s solar + - tox -e pep8 && py.test --cov=solar -s solar services: - riak after_success: diff --git a/solar/cli/base.py b/solar/cli/base.py index d8944047..80c3c1cf 100644 --- a/solar/cli/base.py +++ b/solar/cli/base.py @@ -25,6 +25,7 @@ class AliasedGroup(click.Group): >> solar cha process >> solar res action run rabbitmq_service1 """ + def get_command(self, ctx, cmd_name): rv = click.Group.get_command(self, ctx, cmd_name) if rv is not None: diff --git a/solar/cli/events.py b/solar/cli/events.py index fed3f619..d1510356 100644 --- a/solar/cli/events.py +++ b/solar/cli/events.py @@ -33,7 +33,7 @@ def show(resource): click.echo('Resource: {}'.format(resource)) offset = ' ' * 4 for ev in all_: - click.echo(offset+repr(ev)) + click.echo(offset + repr(ev)) else: click.echo('No events for resource {}'.format(resource)) diff --git a/solar/cli/executors.py b/solar/cli/executors.py index d09ac50e..432a7fad 100644 --- a/solar/cli/executors.py +++ b/solar/cli/executors.py @@ -1,7 +1,22 @@ -from hashlib import md5 +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import hashlib class DryRunExecutor(object): + def __init__(self, mapping=None): from fabric import api as fabric_api from fabric.contrib import project as fabric_project @@ -26,10 +41,11 @@ class DryRunExecutor(object): fabric_api.put = mock.Mock(side_effect=dry_run_executor('PUT')) fabric_api.run = mock.Mock(side_effect=dry_run_executor('SSH RUN')) fabric_api.sudo = mock.Mock(side_effect=dry_run_executor('SSH SUDO')) - fabric_project.rsync_project = mock.Mock(side_effect=dry_run_executor('RSYNC PROJECT')) + fabric_project.rsync_project = mock.Mock( + side_effect=dry_run_executor('RSYNC PROJECT')) def compute_hash(self, key): - return md5(str(key)).hexdigest() + return hashlib.md5(str(key)).hexdigest() def find_hash(self, hash): stripped_hashes = {k.replace('>', ''): k for k in self.mapping} @@ -37,7 +53,7 @@ class DryRunExecutor(object): hashes = [k for k in stripped_hashes if hash.startswith(k)] if len(hashes) == 0: - #raise Exception('Hash {} not found'.format(hash)) + # raise Exception('Hash {} not found'.format(hash)) return '' elif len(hashes) > 1: raise Exception('Hash {} not unique in {}'.format( diff --git a/solar/cli/main.py b/solar/cli/main.py index 9cac54a7..740d055e 100644 --- a/solar/cli/main.py +++ b/solar/cli/main.py @@ -18,30 +18,19 @@ On create "golden" resource should be moved to special place """ import click +import collections from fabric import api as fabric_api import json import networkx as nx -import os -import sys -import tabulate -import yaml -from collections import defaultdict -from solar.core import actions from solar.core import resource as sresource from solar.core import signals -from solar.core.tags_set_parser import Expression -from solar.core.resource import virtual_resource as vr -from solar.core.log import log -from solar import errors -from solar import utils from solar.cli import base -from solar.cli import executors -from solar.cli.orch import orchestration -from solar.cli.system_log import changes from solar.cli.events import events +from solar.cli.orch import orchestration from solar.cli.resource import resource as cli_resource +from solar.cli.system_log import changes # HELPERS @@ -54,7 +43,7 @@ def format_resource_input(resource_name, resource_input): def show_emitter_connections(res): db_obj = res.db_obj - d = defaultdict(list) + d = collections.defaultdict(list) for emitter, receiver, _meta in db_obj.inputs._edges(): d[emitter].append(receiver) @@ -79,7 +68,8 @@ def init_actions(): def run(dry_run_mapping, dry_run, action, tags): raise NotImplementedError("Not yet implemented") # if dry_run: - # dry_run_executor = executors.DryRunExecutor(mapping=json.loads(dry_run_mapping)) + # dry_run_executor = executors.DryRunExecutor( + # mapping=json.loads(dry_run_mapping)) # resources = filter( # lambda r: Expression(tags, r.tags).evaluate(), @@ -94,8 +84,9 @@ def init_actions(): # click.echo('EXECUTED:') # for key in dry_run_executor.executed: # click.echo('{}: {}'.format( - # click.style(dry_run_executor.compute_hash(key), fg='green'), - # str(key) + # click.style(dry_run_executor.compute_hash(key), + # fg='green'), + # str(key) # )) diff --git a/solar/cli/orch.py b/solar/cli/orch.py index 4f28e737..8820d0e2 100755 --- a/solar/cli/orch.py +++ b/solar/cli/orch.py @@ -18,12 +18,13 @@ import time import click +from solar.orchestration import filters from solar.orchestration import graph from solar.orchestration import tasks -from solar.orchestration import filters -from solar.orchestration import utils from solar.orchestration.traversal import states -from solar.cli.uids_history import SOLARUID, remember_uid +from solar.orchestration import utils +from solar.cli.uids_history import remember_uid +from solar.cli.uids_history import SOLARUID from solar import errors @@ -53,11 +54,13 @@ def wait_report(uid, timeout, interval=3): if timeout: for summary in graph.wait_finish(uid, timeout=timeout): stringified_summary = '\r' + ' '.join( - ['{}: {}'.format(state, count) for state, count in summary.items()]) - length = len(stringified_summary) + ['{}: {}'.format(state, count) + for state, count in summary.items()]) click.echo(stringified_summary, nl=False) sys.stdout.flush() - if summary[states.PENDING.name] + summary[states.INPROGRESS.name] != 0: + pending = states.PENDING.name + in_progress = states.INPROGRESS.name + if summary[pending] + summary[in_progress] != 0: time.sleep(interval) except errors.SolarError as err: click.echo('') @@ -83,7 +86,7 @@ def click_report(uid): if item[2]: msg += ' :: {}'.format(item[2]) if item[4] and item[3]: - delta = float(item[4])-float(item[3]) + delta = float(item[4]) - float(item[3]) total += delta msg += ' D: {}'.format(delta) click.echo(click.style(msg, fg=colors[item[1]])) diff --git a/solar/cli/resource.py b/solar/cli/resource.py index d0b8620e..4f07b612 100644 --- a/solar/cli/resource.py +++ b/solar/cli/resource.py @@ -34,6 +34,7 @@ from solar.cli import executors def resource(): pass + @resource.command() @click.argument('action') @click.argument('resource') @@ -41,7 +42,8 @@ def resource(): @click.option('-m', '--dry-run-mapping', default='{}') def action(dry_run_mapping, dry_run, action, resource): if dry_run: - dry_run_executor = executors.DryRunExecutor(mapping=json.loads(dry_run_mapping)) + dry_run_executor = executors.DryRunExecutor( + mapping=json.loads(dry_run_mapping)) click.echo( 'action {} for resource {}'.format(action, resource) @@ -72,13 +74,16 @@ def backtrack_inputs(resource, input, values, real_values): r = sresource.load(resource) db_obj = r.db_obj + def single(resource, name, get_val=False): db_obj = sresource.load(resource).db_obj se = db_obj.inputs._single_edge(name) se = tuple(se) if not se: if get_val: - return dict(resource=resource, name=name, value=db_obj.inputs[name]) + return dict(resource=resource, + name=name, + value=db_obj.inputs[name]) else: return dict(resource=resource, name=name) l = [] @@ -100,7 +105,9 @@ def backtrack_inputs(resource, input, values, real_values): for name, values in inps.iteritems(): click.echo(yaml.safe_dump({name: values}, default_flow_style=False)) if real_values: - click.echo('! Real value: %r' % sresource.load(resource).db_obj.inputs[name] , nl=True) + click.echo('! Real value: %r' % sresource.load( + resource).db_obj.inputs[name], nl=True) + @resource.command() def compile_all(): @@ -111,12 +118,14 @@ def compile_all(): if os.path.exists(destination_path): os.remove(destination_path) - for path in utils.find_by_mask(utils.read_config()['resources-files-mask']): + resources_files_mask = utils.read_config()['resources-files-mask'] + for path in utils.find_by_mask(resources_files_mask): meta = utils.yaml_load(path) meta['base_path'] = os.path.dirname(path) compiler.compile(meta) + @resource.command() def clear_all(): from solar.dblayer.model import ModelMeta @@ -143,6 +152,7 @@ def create(args, base_path, name): for res in resources: click.echo(res.color_repr()) + @resource.command() @click.option('--name', '-n', default=None) @click.option('--tag', '-t', multiple=True) @@ -159,7 +169,8 @@ def show(name, tag, as_json, color): resources = sresource.load_all() if as_json: - output = json.dumps([r.to_dict(inputs=True) for r in resources], indent=2) + output = json.dumps([r.to_dict(inputs=True) + for r in resources], indent=2) echo = click.echo else: if color: @@ -171,6 +182,7 @@ def show(name, tag, as_json, color): if output: echo(output) + @resource.command() @click.argument('resource_name') @click.argument('tags', nargs=-1) @@ -184,6 +196,7 @@ def tag(add, tags, resource_name): r.remove_tags(*tags) click.echo('Tag(s) {} removed from {}'.format(tags, resource_name)) + @resource.command() @click.argument('name') @click.argument('args', nargs=-1) @@ -199,6 +212,7 @@ def update(name, args): res = sresource.load(name) res.update(args_parsed) + @resource.command() @click.option('--check-missing-connections', default=False, is_flag=True) def validate(check_missing_connections): @@ -220,6 +234,7 @@ def validate(check_missing_connections): ]) ) + @resource.command() @click.argument('path', type=click.Path(exists=True, dir_okay=False)) def get_inputs(path): @@ -231,7 +246,8 @@ def get_inputs(path): @resource.command() @click.option('--name', '-n', default=None) @click.option('--tag', '-t', multiple=True) -@click.option('-f', default=False, is_flag=True, help='force removal from database') +@click.option('-f', default=False, is_flag=True, + help='force removal from database') def remove(name, tag, f): if name: resources = [sresource.load(name)] @@ -244,4 +260,5 @@ def remove(name, tag, f): if f: click.echo('Resource %s removed from database' % res.name) else: - click.echo('Resource %s will be removed after commiting changes.' % res.name) + click.echo( + 'Resource %s will be removed after commiting changes.' % res.name) diff --git a/solar/cli/system_log.py b/solar/cli/system_log.py index fc202d6c..21572758 100644 --- a/solar/cli/system_log.py +++ b/solar/cli/system_log.py @@ -48,10 +48,11 @@ def stage(d): click.echo(data.compact(item)) if d: for line in data.details(item.diff): - click.echo(' '*4+line) + click.echo(' ' * 4 + line) if not log: click.echo('No changes') + @changes.command(name='staged-item') @click.argument('uid') def staged_item(uid): @@ -61,7 +62,8 @@ def staged_item(uid): else: click.echo(data.compact(item)) for line in data.details(item.diff): - click.echo(' '*4+line) + click.echo(' ' * 4 + line) + @changes.command() def process(): @@ -90,7 +92,7 @@ def history(n, d, s): click.echo(data.compact(item)) if d: for line in data.details(item.diff): - click.echo(' '*4+line) + click.echo(' ' * 4 + line) if not log: click.echo('No history') @@ -103,6 +105,7 @@ def revert(uid): except errors.SolarError as er: raise click.BadParameter(str(er)) + @changes.command() @click.argument('uids', nargs=-1) @click.option('--all', is_flag=True, default=True) @@ -115,6 +118,7 @@ def discard(uids, all): elif all: change.discard_all() + @changes.command() @click.option('--name', default=None) def test(name): @@ -144,6 +148,7 @@ def test(name): def clean_history(): change.clear_history() + @changes.command(help='USE ONLY FOR TESTING') def commit(): change.commit_all() diff --git a/solar/cli/uids_history.py b/solar/cli/uids_history.py index 1e35a666..56816227 100644 --- a/solar/cli/uids_history.py +++ b/solar/cli/uids_history.py @@ -68,8 +68,9 @@ class SolarUIDParameterType(click.types.StringParamType): try: value = get_uid(value) except IOError: - raise click.BadParameter("Unable to locate file %r so" - "you can't use 'last' shortcuts" % UIDS_HISTORY) + msg = ("Unable to locate file %r so" + "you can't use 'last' shortcuts" % UIDS_HISTORY) + raise click.BadParameter(msg) return value diff --git a/solar/config.py b/solar/config.py index 064744f2..d6f86f61 100644 --- a/solar/config.py +++ b/solar/config.py @@ -1,5 +1,22 @@ +# +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + import os import yaml + from bunch import Bunch CWD = os.getcwd() @@ -26,7 +43,7 @@ def from_configs(): paths = [ os.getenv('SOLAR_CONFIG', os.path.join(CWD, '.config')), os.path.join(CWD, '.config.override') - ] + ] data = {} def _load_from_path(data, path): diff --git a/solar/core/actions.py b/solar/core/actions.py index 65603e38..c311e851 100644 --- a/solar/core/actions.py +++ b/solar/core/actions.py @@ -39,5 +39,5 @@ def resource_action(resource, action): def tag_action(tag, action): - #TODO + # TODO pass diff --git a/solar/core/handlers/__init__.py b/solar/core/handlers/__init__.py index 995fc9c1..530da841 100644 --- a/solar/core/handlers/__init__.py +++ b/solar/core/handlers/__init__.py @@ -27,6 +27,7 @@ HANDLERS = {'ansible': AnsibleTemplate, 'none': Empty, 'puppetv2': PuppetV2} + def get(handler_name): handler = HANDLERS.get(handler_name, None) if handler: diff --git a/solar/core/handlers/ansible_playbook.py b/solar/core/handlers/ansible_playbook.py index 145d533f..28ab038d 100644 --- a/solar/core/handlers/ansible_playbook.py +++ b/solar/core/handlers/ansible_playbook.py @@ -48,14 +48,16 @@ class AnsiblePlaybook(base.BaseHandler): resource.metadata['actions'][action]) stats = callbacks.AggregateStats() playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY) - runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY) + runner_cb = callbacks.PlaybookRunnerCallbacks( + stats, verbose=utils.VERBOSITY) variables = resource.args_dict() if 'roles' in variables: self.download_roles(variables['roles']) remote_user = variables.get('ssh_user') or C.DEFAULT_REMOTE_USER - private_key_file = variables.get('ssh_key') or C.DEFAULT_PRIVATE_KEY_FILE + private_key_file = variables.get( + 'ssh_key') or C.DEFAULT_PRIVATE_KEY_FILE if variables.get('ip'): host = variables['ip'] transport = C.DEFAULT_TRANSPORT diff --git a/solar/core/handlers/ansible_template.py b/solar/core/handlers/ansible_template.py index 5d87416c..0cd91acc 100644 --- a/solar/core/handlers/ansible_template.py +++ b/solar/core/handlers/ansible_template.py @@ -26,7 +26,8 @@ env.warn_only = True # if we would have something like solar_agent that would render this then # we would not need to render it there -# for now we redender it locally, sync to remote, run ansible on remote host as local +# for now we redender it locally, sync to remote, run ansible on remote +# host as local class AnsibleTemplate(TempFileHandler): def action(self, resource, action_name): @@ -42,10 +43,13 @@ class AnsibleTemplate(TempFileHandler): self.transport_sync.sync_all() # remote paths are not nested inside solar_local - remote_playbook_file = playbook_file.replace(SOLAR_TEMP_LOCAL_LOCATION, '/tmp/') - remote_inventory_file = inventory_file.replace(SOLAR_TEMP_LOCAL_LOCATION, '/tmp/') + remote_playbook_file = playbook_file.replace( + SOLAR_TEMP_LOCAL_LOCATION, '/tmp/') + remote_inventory_file = inventory_file.replace( + SOLAR_TEMP_LOCAL_LOCATION, '/tmp/') - call_args = ['ansible-playbook', '--module-path', '/tmp/library', '-i', remote_inventory_file, remote_playbook_file] + call_args = ['ansible-playbook', '--module-path', '/tmp/library', + '-i', remote_inventory_file, remote_playbook_file] log.debug('EXECUTING: %s', ' '.join(call_args)) out = self.transport_run.run(resource, *call_args) diff --git a/solar/core/handlers/base.py b/solar/core/handlers/base.py index 135c802b..3a9b59f4 100644 --- a/solar/core/handlers/base.py +++ b/solar/core/handlers/base.py @@ -49,6 +49,7 @@ class BaseHandler(object): class TempFileHandler(BaseHandler): + def __init__(self, resources, handlers=None): super(TempFileHandler, self).__init__(resources, handlers) self.dst = None @@ -122,7 +123,8 @@ class TempFileHandler(BaseHandler): base_path = resource.db_obj.base_path src_templates_dir = os.path.join(base_path, 'templates') if os.path.exists(src_templates_dir): - trg_templates_dir = os.path.join(self.dirs[resource.name], 'templates') + trg_templates_dir = os.path.join( + self.dirs[resource.name], 'templates') shutil.copytree(src_templates_dir, trg_templates_dir) src_scripts_dir = os.path.join(base_path, 'scripts') @@ -153,5 +155,6 @@ class TempFileHandler(BaseHandler): class Empty(BaseHandler): + def action(self, resource, action): pass diff --git a/solar/core/handlers/puppet.py b/solar/core/handlers/puppet.py index 28f75e23..b83f491f 100644 --- a/solar/core/handlers/puppet.py +++ b/solar/core/handlers/puppet.py @@ -24,8 +24,10 @@ from solar import errors # NOTE: We assume that: # - puppet is installed class Puppet(TempFileHandler): + def action(self, resource, action_name): - log.debug('Executing Puppet manifest %s %s', action_name, resource.name) + log.debug('Executing Puppet manifest %s %s', + action_name, resource.name) action_file = self._compile_action_file(resource, action_name) log.debug('action_file: %s', action_file) diff --git a/solar/core/handlers/python.py b/solar/core/handlers/python.py index ff22af62..d93269f0 100644 --- a/solar/core/handlers/python.py +++ b/solar/core/handlers/python.py @@ -19,6 +19,7 @@ from solar.core.handlers.base import TempFileHandler class Python(TempFileHandler): + def action(self, resource, action_name): action_file = self._compile_action_file(resource, action_name) fabric_api.local('python {}'.format(action_file)) diff --git a/solar/core/handlers/shell.py b/solar/core/handlers/shell.py index bf0024ff..67d689df 100644 --- a/solar/core/handlers/shell.py +++ b/solar/core/handlers/shell.py @@ -21,12 +21,14 @@ from solar.core.handlers.base import TempFileHandler, SOLAR_TEMP_LOCAL_LOCATION class Shell(TempFileHandler): + def action(self, resource, action_name): action_file = self._compile_action_file(resource, action_name) log.debug('action_file: %s', action_file) action_file_name = os.path.join(self.dirs[resource.name], action_file) - action_file_name = action_file_name.replace(SOLAR_TEMP_LOCAL_LOCATION, '/tmp/') + action_file_name = action_file_name.replace( + SOLAR_TEMP_LOCAL_LOCATION, '/tmp/') self._copy_templates_and_scripts(resource, action_name) diff --git a/solar/core/log.py b/solar/core/log.py index e16fe460..20dc2521 100644 --- a/solar/core/log.py +++ b/solar/core/log.py @@ -22,11 +22,13 @@ log = logging.getLogger('solar') def setup_logger(): handler = logging.FileHandler('solar.log') handler.setLevel(logging.DEBUG) - formatter = logging.Formatter('%(asctime)s %(levelname)s %(funcName)s (%(filename)s::%(lineno)s)::%(message)s') + formatter = logging.Formatter( + '%(asctime)s %(levelname)s %(funcName)s (%(filename)s::%(lineno)s)::%(message)s') handler.setFormatter(formatter) log.addHandler(handler) - print_formatter = logging.Formatter('%(levelname)s (%(filename)s::%(lineno)s)::%(message)s') + print_formatter = logging.Formatter( + '%(levelname)s (%(filename)s::%(lineno)s)::%(message)s') print_handler = logging.StreamHandler(stream=sys.stdout) print_handler.setFormatter(print_formatter) log.addHandler(print_handler) diff --git a/solar/core/provider.py b/solar/core/provider.py index 5e6594ab..39c3f4d4 100644 --- a/solar/core/provider.py +++ b/solar/core/provider.py @@ -22,6 +22,7 @@ from solar import utils class BaseProvider(object): + def __init__(self, base_path=None): if base_path is None: self.base_path = utils.read_config()['resources-directory'] @@ -33,6 +34,7 @@ class BaseProvider(object): class DirectoryProvider(BaseProvider): + def __init__(self, directory, *args, **kwargs): self.directory = directory @@ -40,6 +42,7 @@ class DirectoryProvider(BaseProvider): class GitProvider(BaseProvider): + def __init__(self, repository, branch='master', path='.', *args, **kwargs): super(GitProvider, self).__init__(*args, **kwargs) @@ -127,12 +130,14 @@ class SVNProvider(BaseProvider): def __init__(self, url, path='.', base_path=None): self.url = url self.path = path - self.base_path = base_path or utils.read_config()['resources-directory'] + self.base_path = base_path or utils.read_config()[ + 'resources-directory'] if path != '.': self.repo_directory = os.path.join(self.base_path, path) else: self.repo_directory = self.base_path - self.directory = os.path.join(self.repo_directory, self.url.rsplit('/', 1)[-1]) + self.directory = os.path.join( + self.repo_directory, self.url.rsplit('/', 1)[-1]) def run(self): if not os.path.exists(self.repo_directory): diff --git a/solar/core/resource/resource.py b/solar/core/resource/resource.py index 81720a18..5a6bf02a 100644 --- a/solar/core/resource/resource.py +++ b/solar/core/resource/resource.py @@ -51,7 +51,8 @@ def read_meta(base_path): return metadata -RESOURCE_STATE = Enum('ResourceState', 'created operational removed error updated') +RESOURCE_STATE = Enum( + 'ResourceState', 'created operational removed error updated') class Resource(object): @@ -100,7 +101,6 @@ class Resource(object): self.db_obj.save() - # Load @dispatch(DBResource) def __init__(self, resource_db): @@ -117,7 +117,7 @@ class Resource(object): inputs.setdefault('location_id', {'value': "", 'schema': 'str!'}) inputs.setdefault('transports_id', {'value': "", - 'schema': 'str'}) + 'schema': 'str'}) for inp in ('transports_id', 'location_id'): if inputs[inp].get('value') == '$uuid': inputs[inp]['value'] = md5(self.name + uuid4().hex).hexdigest() @@ -222,7 +222,7 @@ class Resource(object): for (emitter_resource, emitter_input), (receiver_resource, receiver_input), meta in self.graph().edges(data=True): if meta: receiver_input = '{}:{}|{}'.format(receiver_input, - meta['destination_key'], meta['tag']) + meta['destination_key'], meta['tag']) rst.add( (emitter_resource, emitter_input, @@ -269,9 +269,8 @@ class Resource(object): self.db_obj.save_lazy() receiver.db_obj.save_lazy() - def connect_with_events(self, receiver, mapping=None, events=None, - use_defaults=False): + use_defaults=False): mapping = get_mapping(self, receiver, mapping) self._connect_inputs(receiver, mapping) # signals.connect(self, receiver, mapping=mapping) @@ -292,7 +291,6 @@ class Resource(object): self.db_obj.save_lazy() - def load(name): r = DBResource.get(name) @@ -313,6 +311,8 @@ def load_updated(since=None, with_childs=True): return [Resource(r) for r in DBResource.multi_get(candids)] # TODO + + def load_all(): candids = DBResource.updated.filter(StrInt.p_min(), StrInt.p_max()) return [Resource(r) for r in DBResource.multi_get(candids)] diff --git a/solar/core/resource/virtual_resource.py b/solar/core/resource/virtual_resource.py index 1b3d0a7a..de31f257 100644 --- a/solar/core/resource/virtual_resource.py +++ b/solar/core/resource/virtual_resource.py @@ -109,7 +109,8 @@ def _get_template(name, content, kwargs, inputs): if input not in kwargs: missing.append(input) if missing: - raise Exception('[{0}] Validation error. Missing data in input: {1}'.format(name, missing)) + raise Exception( + '[{0}] Validation error. Missing data in input: {1}'.format(name, missing)) template = Template(content, trim_blocks=True, lstrip_blocks=True) template = template.render(str=str, zip=zip, **kwargs) return template @@ -158,6 +159,7 @@ def extend_resources(template_resources): log.debug('Warrning: no resources with tags: {}'.format(tags)) return resources + def update_resources(template_resources): resources = extend_resources(template_resources) for r in resources: @@ -197,7 +199,7 @@ def extend_events(template_events): resources = load_by_tags(tags) for r in resources: parent_action = '{}.{}'.format(r.name, parent['action']) - event = {'type' : e['type'], + event = {'type': e['type'], 'state': e['state'], 'depend_action': e['depend_action'], 'parent_action': parent_action @@ -205,6 +207,7 @@ def extend_events(template_events): events.append(event) return events + def parse_events(template_events): parsed_events = [] events = extend_events(template_events) @@ -223,8 +226,6 @@ def parse_events(template_events): return parsed_events - - def parse_inputs(args): connections = [] assignments = {} @@ -272,7 +273,7 @@ def parse_connection(child_input, element): except ValueError: events = None return {'child_input': child_input, - 'parent' : parent, + 'parent': parent, 'parent_input': parent_input, - 'events' : events + 'events': events } diff --git a/solar/core/signals.py b/solar/core/signals.py index 40492867..897c8108 100644 --- a/solar/core/signals.py +++ b/solar/core/signals.py @@ -81,7 +81,8 @@ def location_and_transports(emitter, receiver, orig_mapping): receiver_single_reverse = receiver_single.get('reverse') if inps_receiver is None and inps_emitter is not None: # we don't connect automaticaly when receiver is None and emitter is not None - # for cases when we connect existing transports to other data containers + # for cases when we connect existing transports to other data + # containers if receiver_single_reverse: log.info("Didn't connect automaticaly %s::%s -> %s::%s", receiver.name, @@ -124,7 +125,8 @@ def location_and_transports(emitter, receiver, orig_mapping): # with dirty_state_ok(DBResource, ('index', )): for single in ('transports_id', 'location_id'): if single in inps_emitter and single in inps_receiver: - _single(single, emitter, receiver, inps_emitter[single], inps_receiver[single]) + _single(single, emitter, receiver, inps_emitter[ + single], inps_receiver[single]) else: log.warning('Unable to create connection for %s with' ' emitter %s, receiver %s', diff --git a/solar/core/tags_set_parser.py b/solar/core/tags_set_parser.py index 0a48b02a..a805eb1a 100644 --- a/solar/core/tags_set_parser.py +++ b/solar/core/tags_set_parser.py @@ -26,10 +26,10 @@ tokens = ( "RPAREN") t_STRING = r'[A-Za-z0-9-_/\\]+' -t_AND = '&|,' -t_OR = r'\|' -t_LPAREN = r'\(' -t_RPAREN = r'\)' +t_AND = '&|,' +t_OR = r'\|' +t_LPAREN = r'\(' +t_RPAREN = r'\)' t_ignore = ' \t\r\n' @@ -48,6 +48,7 @@ class SubexpressionWrapper(object): class ScalarWrapper(object): + def __init__(self, value): global expression self.value = (set([value]) <= set(expression.tags)) @@ -90,10 +91,12 @@ def t_error(t): def p_error(p): - raise errors.ParseError("Syntax error at '{0}'".format(getattr(p, 'value', ''))) + raise errors.ParseError( + "Syntax error at '{0}'".format(getattr(p, 'value', ''))) class Expression(object): + def __init__(self, expression_text, tags): self.expression_text = expression_text self.tags = tags diff --git a/solar/core/transports/base.py b/solar/core/transports/base.py index 90016ff6..639934fb 100644 --- a/solar/core/transports/base.py +++ b/solar/core/transports/base.py @@ -90,7 +90,8 @@ class SolarTransport(object): except AttributeError: if name is None: name = self.preffered_transport_name - transport = next(x for x in resource.transports() if x['name'] == name) + transport = next(x for x in resource.transports() + if x['name'] == name) setattr(resource, key, transport) return transport diff --git a/solar/core/transports/bat.py b/solar/core/transports/bat.py index b21e9715..7a9c8f0b 100644 --- a/solar/core/transports/bat.py +++ b/solar/core/transports/bat.py @@ -63,7 +63,8 @@ class BatTransport(SolarTransport): except AttributeError: transports = resource.transports() for pref in self._order: - selected = next((x for x in transports if x['name'] == pref), None) + selected = next( + (x for x in transports if x['name'] == pref), None) if selected: break if not selected: diff --git a/solar/core/transports/helpers/solar_torrent.py b/solar/core/transports/helpers/solar_torrent.py index 2f00f2ad..bbd248c7 100644 --- a/solar/core/transports/helpers/solar_torrent.py +++ b/solar/core/transports/helpers/solar_torrent.py @@ -7,7 +7,7 @@ import time import sys import os -state_str = ['queued', 'checking', 'downloading metadata', \ +state_str = ['queued', 'checking', 'downloading metadata', 'downloading', 'finished', 'seeding', 'allocating', 'checking fastresume'] @@ -31,7 +31,8 @@ class MultiTorrent(object): @property def progress(self): - total_progress = map(attrgetter('progress'), map(lambda x: x.status(), self.torrents)) + total_progress = map(attrgetter('progress'), map( + lambda x: x.status(), self.torrents)) return sum(total_progress) / len(total_progress) def numbers(self): @@ -52,10 +53,10 @@ def init_session(args, seed=False): if os.path.exists(magnet_or_path): e = lt.bdecode(open(magnet_or_path, 'rb').read()) info = lt.torrent_info(e) - params = { 'save_path': save_path, - 'storage_mode': lt.storage_mode_t.storage_mode_sparse, - 'ti': info, - 'seed_mode': seed} + params = {'save_path': save_path, + 'storage_mode': lt.storage_mode_t.storage_mode_sparse, + 'ti': info, + 'seed_mode': seed} h = ses.add_torrent(params) else: h = ses.add_torrent({ @@ -119,7 +120,6 @@ def _seeder(torrents, save_path='.', max_seed_ratio=5): sys.exit(0) - def _getter(torrents, max_seed_ratio=3): ses = lt.session() ses.listen_on(6881, 6981) @@ -137,7 +137,7 @@ def _getter(torrents, max_seed_ratio=3): s = ses.status() if i % 5 == 0: print '%.2f%% complete (down: %.1f kb/s up: %.1f kB/s peers: %d) %s' % \ - (mt.progress * 100, s.download_rate / 1000, s.upload_rate / 1000, \ + (mt.progress * 100, s.download_rate / 1000, s.upload_rate / 1000, s.num_peers, mt.numbers()) now = time.time() current_state = (now, mt.progress) diff --git a/solar/core/transports/solar_agent_transport.py b/solar/core/transports/solar_agent_transport.py index 985b2c3a..5787ca42 100644 --- a/solar/core/transports/solar_agent_transport.py +++ b/solar/core/transports/solar_agent_transport.py @@ -29,8 +29,8 @@ class SolarAgentTransport(object): auth = transport['password'] transport_class = transport.get('transport_class') client = SolarAgentClient(auth={'user': user, 'auth': auth}, - transport_args=(host, port), - transport_class=transport_class) + transport_args=(host, port), + transport_class=transport_class) return client @@ -61,4 +61,3 @@ class SolarAgentRunTransport(RunTransport, SolarAgentTransport): client = self.get_client(resource) res = client.run(' '.join(args), **kwargs) return self.get_result(res) - diff --git a/solar/core/transports/ssh_raw.py b/solar/core/transports/ssh_raw.py index 92773a1e..a5b17395 100644 --- a/solar/core/transports/ssh_raw.py +++ b/solar/core/transports/ssh_raw.py @@ -33,7 +33,6 @@ class _RawSSHTransport(object): return ('ssh', '-i', props['ssh_key']) - class RawSSHRunTransport(RunTransport, _RawSSHTransport): def run(self, resource, *args, **kwargs): @@ -59,4 +58,3 @@ class RawSSHRunTransport(RunTransport, _RawSSHTransport): log.debug("SSH CMD: %r", ssh_cmd) return fabric_api.local(' '.join(ssh_cmd)) - diff --git a/solar/core/transports/torrent.py b/solar/core/transports/torrent.py index b27ab425..a4b98e0d 100644 --- a/solar/core/transports/torrent.py +++ b/solar/core/transports/torrent.py @@ -45,7 +45,8 @@ class TorrentSyncTransport(SyncTransport): def _create_torrent(self, resource, fs, root='.', use_sudo=False): t = lt.create_torrent(fs) transports = resource.transports() - torrent_transport = next((x for x in transports if x['name'] == 'torrent')) + torrent_transport = next( + (x for x in transports if x['name'] == 'torrent')) trackers = torrent_transport['trackers'] for tracker in trackers: t.add_tracker(tracker) @@ -77,7 +78,8 @@ class TorrentSyncTransport(SyncTransport): # we don't need use sudo there for now from fabric import api as fabric_api torrents = self._torrents + self._sudo_torrents - to_seed = ["%s|%s" % (os.path.abspath(os.path.join(x[2], '..')), x[0]) for x in torrents] + to_seed = ["%s|%s" % (os.path.abspath( + os.path.join(x[2], '..')), x[0]) for x in torrents] seed_args = ';'.join(to_seed) # TODO: 'g' is just for debug, it should be 's', remove when sure cmd = ['/usr/bin/python', @@ -95,7 +97,8 @@ class TorrentSyncTransport(SyncTransport): torrents = self._torrents else: torrents = self._sudo_torrents - to_get = ["%s|%s" % (os.path.abspath(os.path.join(x[2], '..')), x[1]) for x in torrents] + to_get = ["%s|%s" % (os.path.abspath( + os.path.join(x[2], '..')), x[1]) for x in torrents] get_args = ';'.join(to_get) cmd = ['/usr/bin/python', '/var/tmp/solar_torrent.py', @@ -115,4 +118,3 @@ class TorrentSyncTransport(SyncTransport): self._start_remote_fetch(resource, use_sudo=False) if self._sudo_torrents: self._start_remote_fetch(resource, use_sudo=True) - diff --git a/solar/core/validation.py b/solar/core/validation.py index 59d5018d..6932af5f 100644 --- a/solar/core/validation.py +++ b/solar/core/validation.py @@ -62,8 +62,6 @@ import requests from solar.core.log import log - - def schema_input_type(schema): """Input type from schema @@ -99,7 +97,8 @@ def _construct_jsonschema(schema, definition_base=''): return {'type': 'boolean'}, {} if isinstance(schema, list): - items, definitions = _construct_jsonschema(schema[0], definition_base=definition_base) + items, definitions = _construct_jsonschema( + schema[0], definition_base=definition_base) return { 'type': 'array', @@ -114,9 +113,11 @@ def _construct_jsonschema(schema, definition_base=''): if isinstance(v, dict) or isinstance(v, list): key = '{}_{}'.format(definition_base, k) properties[k] = {'$ref': '#/definitions/{}'.format(key)} - definitions[key], new_definitions = _construct_jsonschema(v, definition_base=key) + definitions[key], new_definitions = _construct_jsonschema( + v, definition_base=key) else: - properties[k], new_definitions = _construct_jsonschema(v, definition_base=definition_base) + properties[k], new_definitions = _construct_jsonschema( + v, definition_base=definition_base) definitions.update(new_definitions) @@ -177,7 +178,7 @@ def validate_resource(r): for input_name, _ in inputs.items(): errors = validate_input( args.get(input_name), - #jsonschema=input_definition.get('jsonschema'), + # jsonschema=input_definition.get('jsonschema'), schema=r.db_obj.meta_inputs[input_name]['schema'] ) if errors: diff --git a/solar/events/api.py b/solar/events/api.py index bd002e40..a814cb0f 100644 --- a/solar/events/api.py +++ b/solar/events/api.py @@ -22,6 +22,7 @@ from solar.events.controls import Dep, React, StateChange from solar.dblayer.solar_models import Resource + def create_event(event_dict): etype = event_dict['etype'] kwargs = {'child': event_dict['child'], @@ -125,7 +126,6 @@ def bft_events_graph(start): return dg - def build_edges(changes_graph, events): """ :param changes_graph: nx.DiGraph object with actions to be executed @@ -143,7 +143,8 @@ def build_edges(changes_graph, events): event_name = stack.pop(0) if event_name in events_graph: - log.debug('Next events after %s are %s', event_name, events_graph.successors(event_name)) + log.debug('Next events after %s are %s', event_name, + events_graph.successors(event_name)) else: log.debug('No outgoing events based on %s', event_name) diff --git a/solar/events/controls.py b/solar/events/controls.py index 6d204de0..2072fcc6 100644 --- a/solar/events/controls.py +++ b/solar/events/controls.py @@ -34,6 +34,7 @@ trigger action even if no changes noticed on dependent resource. from solar.dblayer.solar_models import Resource from solar.dblayer.model import DBLayerNotFound + class Event(object): etype = None @@ -84,12 +85,13 @@ class Dependency(Event): def insert(self, changed_resources, changes_graph): if (self.parent_node in changes_graph and - self.child_node in changes_graph): + self.child_node in changes_graph): changes_graph.add_edge( self.parent_node, self.child_node, state=self.state) Dep = Dependency + class React(Event): etype = 'react_on' @@ -99,7 +101,8 @@ class React(Event): if self.parent_node in changes_graph: if self.child_node not in changes_graph: try: - location_id = Resource.get(self.child).inputs['location_id'] + location_id = Resource.get(self.child).inputs[ + 'location_id'] except DBLayerNotFound: location_id = None changes_graph.add_node( diff --git a/solar/orchestration/consts.py b/solar/orchestration/consts.py index b28b04f6..e69de29b 100644 --- a/solar/orchestration/consts.py +++ b/solar/orchestration/consts.py @@ -1,3 +0,0 @@ - - - diff --git a/solar/orchestration/filters.py b/solar/orchestration/filters.py index 51ecc397..ea1cf793 100644 --- a/solar/orchestration/filters.py +++ b/solar/orchestration/filters.py @@ -44,7 +44,6 @@ def start_from(dg, start_nodes): if not preds and node in start_nodes: visited.add(node) - if preds: for pred in preds: if pred not in visited: @@ -63,9 +62,11 @@ def validate(dg, start_nodes, end_nodes, err_msgs): for n in end_nodes: if n not in dg: if start_nodes: - error_msgs.append('No path from {} to {}'.format(start_nodes, n)) + error_msgs.append( + 'No path from {} to {}'.format(start_nodes, n)) else: - error_msgs.append(not_in_the_graph_msg.format(n, dg.graph['uid'])) + error_msgs.append( + not_in_the_graph_msg.format(n, dg.graph['uid'])) return error_msgs diff --git a/solar/orchestration/graph.py b/solar/orchestration/graph.py index 28e88047..50670e30 100644 --- a/solar/orchestration/graph.py +++ b/solar/orchestration/graph.py @@ -86,7 +86,7 @@ def parse_plan(plan_path): defaults = { 'status': 'PENDING', 'errmsg': '', - } + } defaults.update(task['parameters']) dg.add_node( task['uid'], **defaults) @@ -129,7 +129,6 @@ def create_plan(plan_path, save=True): return create_plan_from_graph(dg, save=save) - def reset_by_uid(uid, state_list=None): dg = get_graph(uid) return reset(dg, state_list=state_list) diff --git a/solar/orchestration/limits.py b/solar/orchestration/limits.py index cae4f6c6..6c19f340 100644 --- a/solar/orchestration/limits.py +++ b/solar/orchestration/limits.py @@ -51,8 +51,10 @@ def type_based_rule(dg, inprogress, item): type_limit: 2 """ _type = dg.node[item].get('resource_type') - if 'type_limit' not in dg.node[item]: return True - if not _type: return True + if 'type_limit' not in dg.node[item]: + return True + if not _type: + return True type_count = 0 for n in inprogress: @@ -63,7 +65,8 @@ def type_based_rule(dg, inprogress, item): def target_based_rule(dg, inprogress, item, limit=1): target = dg.node[item].get('target') - if not target: return True + if not target: + return True target_count = 0 for n in inprogress: diff --git a/solar/orchestration/runner.py b/solar/orchestration/runner.py index 448a30f1..c8c986eb 100644 --- a/solar/orchestration/runner.py +++ b/solar/orchestration/runner.py @@ -22,5 +22,5 @@ app = Celery( include=['solar.system_log.tasks', 'solar.orchestration.tasks'], backend=_url, broker=_url) -app.conf.update(CELERY_ACCEPT_CONTENT = ['json']) -app.conf.update(CELERY_TASK_SERIALIZER = 'json') +app.conf.update(CELERY_ACCEPT_CONTENT=['json']) +app.conf.update(CELERY_TASK_SERIALIZER='json') diff --git a/solar/orchestration/tasks.py b/solar/orchestration/tasks.py index 3f2d005e..600734a7 100644 --- a/solar/orchestration/tasks.py +++ b/solar/orchestration/tasks.py @@ -54,10 +54,12 @@ class ReportTask(task.Task): report_task = partial(app.task, base=ReportTask, bind=True) + @task_prerun.connect def start_solar_session(task_id, task, *args, **kwargs): ModelMeta.session_start() + @task_postrun.connect def end_solar_session(task_id, task, *args, **kwargs): ModelMeta.session_end() @@ -104,7 +106,7 @@ def fault_tolerance(ctxt, percent): if dg.node[s]['status'] == 'SUCCESS': success += 1 - succes_percent = (success/lth) * 100 + succes_percent = (success / lth) * 100 if succes_percent < percent: raise Exception('Cant proceed with, {0} < {1}'.format( succes_percent, percent)) @@ -117,7 +119,8 @@ def echo(ctxt, message): @report_task(name='anchor') def anchor(ctxt, *args): - # such tasks should be walked when atleast 1/3/exact number of resources visited + # such tasks should be walked when atleast 1/3/exact number of resources + # visited dg = graph.get_graph('current') for s in dg.predecessors(ctxt.request.id): if dg.node[s]['status'] != 'SUCCESS': @@ -155,6 +158,7 @@ def soft_stop(plan_uid): dg.node[n]['status'] = 'SKIPPED' graph.update_graph(dg) + @app.task(name='schedule_next') def schedule_next(task_id, status, errmsg=None): plan_uid, task_name = task_id.rsplit(':', 1) diff --git a/solar/orchestration/utils.py b/solar/orchestration/utils.py index a694f49c..375b5969 100644 --- a/solar/orchestration/utils.py +++ b/solar/orchestration/utils.py @@ -34,5 +34,6 @@ def write_graph(plan): nx.write_dot(plan, '{name}.dot'.format(name=plan.graph['name'])) subprocess.call( - 'tred {name}.dot | dot -Tsvg -o {name}.svg'.format(name=plan.graph['name']), + 'tred {name}.dot | dot -Tsvg -o {name}.svg'.format( + name=plan.graph['name']), shell=True) diff --git a/solar/system_log/change.py b/solar/system_log/change.py index 6d57cf36..46107fdc 100644 --- a/solar/system_log/change.py +++ b/solar/system_log/change.py @@ -29,6 +29,7 @@ from solar.errors import CannotFindID from solar.dblayer.solar_models import Resource, LogItem, CommitedResource, StrInt + def guess_action(from_, to): # NOTE(dshulyak) imo the way to solve this - is dsl for orchestration, # something where this action will be excplicitly specified @@ -47,12 +48,12 @@ def create_diff(staged, commited): def create_logitem(resource, action, diffed, connections_diffed, base_path=''): return LogItem.new( - {'resource': resource, - 'action': action, - 'diff': diffed, - 'connections_diff': connections_diffed, - 'base_path': base_path, - 'log': 'staged'}) + {'resource': resource, + 'action': action, + 'diff': diffed, + 'connections_diff': connections_diffed, + 'base_path': base_path, + 'log': 'staged'}) def create_sorted_diff(staged, commited): @@ -104,7 +105,7 @@ def stage_changes(): last = LogItem.history_last() since = StrInt.greater(last.updated) if last else None staged_log = utils.solar_map(make_single_stage_item, - resource.load_updated(since), concurrency=10) + resource.load_updated(since), concurrency=10) staged_log = filter(None, staged_log) return staged_log @@ -139,9 +140,10 @@ def _get_args_to_update(args, connections): """ inherited = [i[3].split(':')[0] for i in connections] return { - key:args[key] for key in args + key: args[key] for key in args if key not in inherited - } + } + def revert_uids(uids): """ @@ -167,14 +169,16 @@ def _revert_remove(logitem): """ commited = CommitedResource.get(logitem.resource) args = dictdiffer.revert(logitem.diff, commited.inputs) - connections = dictdiffer.revert(logitem.connections_diff, sorted(commited.connections)) + connections = dictdiffer.revert( + logitem.connections_diff, sorted(commited.connections)) resource.Resource(logitem.resource, logitem.base_path, - args=_get_args_to_update(args, connections), tags=commited.tags) + args=_get_args_to_update(args, connections), tags=commited.tags) for emitter, emitter_input, receiver, receiver_input in connections: emmiter_obj = resource.load(emitter) receiver_obj = resource.load(receiver) - signals.connect(emmiter_obj, receiver_obj, {emitter_input: receiver_input}) + signals.connect(emmiter_obj, receiver_obj, { + emitter_input: receiver_input}) def _update_inputs_connections(res_obj, args, old_connections, new_connections): @@ -213,7 +217,8 @@ def _revert_update(logitem): res_obj = resource.load(logitem.resource) commited = res_obj.load_commited() - connections = dictdiffer.revert(logitem.connections_diff, sorted(commited.connections)) + connections = dictdiffer.revert( + logitem.connections_diff, sorted(commited.connections)) args = dictdiffer.revert(logitem.diff, commited.inputs) _update_inputs_connections( @@ -237,12 +242,14 @@ def _discard_remove(item): def _discard_update(item): resource_obj = resource.load(item.resource) old_connections = resource_obj.connections - new_connections = dictdiffer.revert(item.connections_diff, sorted(old_connections)) + new_connections = dictdiffer.revert( + item.connections_diff, sorted(old_connections)) args = dictdiffer.revert(item.diff, resource_obj.args) _update_inputs_connections( resource_obj, _get_args_to_update(args, new_connections), old_connections, new_connections) + def _discard_run(item): resource.load(item.resource).remove(force=True) @@ -265,6 +272,7 @@ def discard_uids(uids): def discard_uid(uid): return discard_uids([uid]) + def discard_all(): staged_log = data.SL() return discard_uids([l.uid for l in staged_log]) @@ -277,6 +285,7 @@ def commit_all(): for item in data.SL(): move_to_commited(item.log_action) + def clear_history(): LogItem.delete_all() CommitedResource.delete_all() diff --git a/solar/system_log/consts.py b/solar/system_log/consts.py index af2d5f76..77d6f4e3 100644 --- a/solar/system_log/consts.py +++ b/solar/system_log/consts.py @@ -15,9 +15,9 @@ from enum import Enum CHANGES = Enum( - 'Changes', - 'run remove update' - ) + 'Changes', + 'run remove update' +) STATES = Enum('States', 'error inprogress pending success') diff --git a/solar/system_log/data.py b/solar/system_log/data.py index d4561978..db0cddcf 100644 --- a/solar/system_log/data.py +++ b/solar/system_log/data.py @@ -16,17 +16,16 @@ from solar.dblayer.solar_models import LogItem - def SL(): rst = LogItem.composite.filter({'log': 'staged'}) return LogItem.multi_get(rst) + def CL(): rst = LogItem.composite.filter({'log': 'history'}) return LogItem.multi_get(rst) - def compact(logitem): return 'log task={} uid={}'.format(logitem.log_action, logitem.uid) @@ -36,13 +35,13 @@ def details(diff): for type_, val, change in diff: if type_ == 'add': for key, val in change: - rst.append('++ {}: {}'.format(key ,val)) + rst.append('++ {}: {}'.format(key, val)) elif type_ == 'change': rst.append('-+ {}: {} >> {}'.format( unwrap_change_val(val), change[0], change[1])) elif type_ == 'remove': for key, val in change: - rst.append('-- {}: {}'.format(key ,val)) + rst.append('-- {}: {}'.format(key, val)) return rst @@ -62,4 +61,3 @@ def unwrap_change_val(val): return '{}:[{}] '.format(val[0], val[1]) else: return val - diff --git a/solar/template.py b/solar/template.py index 7ce42e01..f7f7b48f 100644 --- a/solar/template.py +++ b/solar/template.py @@ -198,7 +198,8 @@ class ResourceListTemplate(BaseTemplate): self.add_react(action_state, ResourceTemplate(r), action) def filter(self, func): - """Return ResourceListeTemplate instance with resources filtered by func. + """Return ResourceListeTemplate instance with resources filtered + by func. func -- predictate function that takes (idx, resource) as parameter (idx is the index of resource in self.resources list) @@ -233,7 +234,8 @@ class ResourceListTemplate(BaseTemplate): ) def connect_list_to_each(self, resources, mapping=None, events=None): - """Connect each resource in self.resources to each resource in resources. + """Connect each resource in self.resources to each resource in + resources. mapping -- optional mapping "{emitter_num}" -- substitutes for emitter's index in mapping (from @@ -260,7 +262,8 @@ class ResourceListTemplate(BaseTemplate): ) def on_each(self, resource_path, args=None): - """Create resource form resource_path on each resource in self.resources. + """Create resource form resource_path on each resource in + self.resources. """ args = args or {} diff --git a/solar/test/conftest.py b/solar/test/conftest.py index 9240c582..fff7d599 100644 --- a/solar/test/conftest.py +++ b/solar/test/conftest.py @@ -23,6 +23,7 @@ import pytest def patched_get_bucket_name(cls): return cls.__name__ + str(time.time()) + @pytest.fixture def resources(): base_path = os.path.join( @@ -30,15 +31,16 @@ def resources(): 'resource_fixtures') node_path = os.path.join(base_path, 'node') - node1 = Resource('node1', node_path, args={'ip':'10.0.0.1'}) - node2 = Resource('node2', node_path, args={'ip':'10.0.0.2'}) + node1 = Resource('node1', node_path, args={'ip': '10.0.0.1'}) + node2 = Resource('node2', node_path, args={'ip': '10.0.0.2'}) base_service_path = os.path.join(base_path, 'base_service') service1 = Resource('service1', base_service_path) - return {'node1' : node1, - 'node2' : node2, + return {'node1': node1, + 'node2': node2, 'service1': service1 - } + } + @pytest.fixture(autouse=True) def setup(request): @@ -53,15 +55,20 @@ def setup(request): for model in ModelMeta._defined_models: model.bucket = get_bucket(None, model, ModelMeta) + def pytest_runtest_teardown(item, nextitem): ModelMeta.session_end(result=True) return nextitem # It will run before all fixtures + + def pytest_runtest_setup(item): ModelMeta.session_start() # it will run after fixtures but before test + + def pytest_runtest_call(item): ModelMeta.session_end() ModelMeta.session_start() diff --git a/solar/test/test_diff_generation.py b/solar/test/test_diff_generation.py index 1df1e67c..6f09204f 100644 --- a/solar/test/test_diff_generation.py +++ b/solar/test/test_diff_generation.py @@ -24,24 +24,26 @@ def staged(): return {'id': 'res.1', 'tags': ['res', 'node.1'], 'input': {'ip': {'value': '10.0.0.2'}, - 'list_val': {'value': [1, 2]}}, + 'list_val': {'value': [1, 2]}}, 'metadata': {}, 'connections': [ ['node.1', 'res.1', ['ip', 'ip']], ['node.1', 'res.1', ['key', 'key']]] } + @fixture def commited(): return {'id': 'res.1', 'tags': ['res', 'node.1'], 'input': {'ip': '10.0.0.2', - 'list_val': [1]}, + 'list_val': [1]}, 'metadata': {}, 'connections': [ ['node.1', 'res.1', ['ip', 'ip']]] } + @fixture def full_diff(staged): return change.create_diff(staged, {}) @@ -54,7 +56,8 @@ def diff_for_update(staged, commited): def test_create_diff_with_empty_commited(full_diff): # add will be executed - expected = [('add', '', [('connections', [['node.1', 'res.1', ['ip', 'ip']], ['node.1', 'res.1', ['key', 'key']]]), ('input', {'ip': {'value': '10.0.0.2'}, 'list_val': {'value': [1, 2]}}), ('metadata', {}), ('id', 'res.1'), ('tags', ['res', 'node.1'])])] + expected = [('add', '', [('connections', [['node.1', 'res.1', ['ip', 'ip']], ['node.1', 'res.1', ['key', 'key']]]), ('input', { + 'ip': {'value': '10.0.0.2'}, 'list_val': {'value': [1, 2]}}), ('metadata', {}), ('id', 'res.1'), ('tags', ['res', 'node.1'])])] assert full_diff == expected @@ -62,8 +65,8 @@ def test_create_diff_modified(diff_for_update): assert diff_for_update == [ ('add', 'connections', [(1, ['node.1', 'res.1', ['key', 'key']])]), - ('change', 'input.ip', ('10.0.0.2', {'value': '10.0.0.2'})), - ('change', 'input.list_val', ([1], {'value': [1, 2]}))] + ('change', 'input.ip', ('10.0.0.2', {'value': '10.0.0.2'})), + ('change', 'input.list_val', ([1], {'value': [1, 2]}))] def test_verify_patch_creates_expected(staged, diff_for_update, commited): @@ -79,7 +82,7 @@ def test_revert_update(staged, diff_for_update, commited): @fixture def resources(): r = {'n.1': - {'uid': 'n.1', + {'uid': 'n.1', 'args': {'ip': '10.20.0.2'}, 'connections': [], 'tags': []}, @@ -88,7 +91,7 @@ def resources(): 'args': {'ip': '10.20.0.2'}, 'connections': [['n.1', 'r.1', ['ip', 'ip']]], 'tags': []}, - 'h.1': + 'h.1': {'uid': 'h.1', 'args': {'ip': '10.20.0.2', 'ips': ['10.20.0.2']}, diff --git a/solar/test/test_events.py b/solar/test/test_events.py index 84860b26..1ef78cb0 100644 --- a/solar/test/test_events.py +++ b/solar/test/test_events.py @@ -99,7 +99,8 @@ def rmq_deps(): 'rmq.2': [evapi.Dep('rmq.2', 'run', 'success', 'rmq_cluster.2', 'join')], 'rmq.3': [evapi.Dep('rmq.3', 'run', 'success', 'rmq_cluster.3', 'join')], 'rmq_cluster.1': [ - evapi.Dep('rmq_cluster.1', 'create', 'success', 'rmq_cluster.2', 'join'), + evapi.Dep('rmq_cluster.1', 'create', + 'success', 'rmq_cluster.2', 'join'), evapi.Dep('rmq_cluster.1', 'create', 'success', 'rmq_cluster.3', 'join')]} @@ -121,13 +122,16 @@ def test_riak(): events = { 'riak_service1': [ - evapi.React('riak_service1', 'run', 'success', 'riak_service2', 'run'), + evapi.React('riak_service1', 'run', 'success', + 'riak_service2', 'run'), evapi.React('riak_service1', 'run', 'success', 'riak_service3', 'run')], 'riak_service3': [ - evapi.React('riak_service3', 'join', 'success', 'riak_service1', 'commit'), + evapi.React('riak_service3', 'join', 'success', + 'riak_service1', 'commit'), evapi.React('riak_service3', 'run', 'success', 'riak_service3', 'join')], 'riak_service2': [ - evapi.React('riak_service2', 'run', 'success', 'riak_service2', 'join'), + evapi.React('riak_service2', 'run', 'success', + 'riak_service2', 'join'), evapi.React('riak_service2', 'join', 'success', 'riak_service1', 'commit')], } @@ -135,4 +139,5 @@ def test_riak(): changes_graph = nx.MultiDiGraph() changes_graph.add_node('riak_service1.run') evapi.build_edges(changes_graph, events) - assert set(changes_graph.predecessors('riak_service1.commit')) == {'riak_service2.join', 'riak_service3.join'} + assert set(changes_graph.predecessors('riak_service1.commit')) == { + 'riak_service2.join', 'riak_service3.join'} diff --git a/solar/test/test_graph_api.py b/solar/test/test_graph_api.py index 65c488ce..94f10a17 100644 --- a/solar/test/test_graph_api.py +++ b/solar/test/test_graph_api.py @@ -34,6 +34,7 @@ def test_simple_plan_created_and_loaded(simple): plan = graph.get_plan(simple.graph['uid']) assert set(plan.nodes()) == {'just_fail', 'echo_stuff'} + def test_reset_all_states(simple): for n in simple: simple.node[n]['status'] == states.ERROR.name @@ -57,16 +58,19 @@ def test_wait_finish(simple): for n in simple: simple.node[n]['status'] = states.SUCCESS.name graph.update_graph(simple) - assert next(graph.wait_finish(simple.graph['uid'], 10)) == {'SKIPPED': 0, 'SUCCESS': 2, 'NOOP': 0, 'ERROR': 0, 'INPROGRESS': 0, 'PENDING': 0} + assert next(graph.wait_finish(simple.graph['uid'], 10)) == { + 'SKIPPED': 0, 'SUCCESS': 2, 'NOOP': 0, 'ERROR': 0, 'INPROGRESS': 0, 'PENDING': 0} def test_several_updates(simple): simple.node['just_fail']['status'] = states.ERROR.name graph.update_graph(simple) - assert next(graph.wait_finish(simple.graph['uid'], 10)) == {'SKIPPED': 0, 'SUCCESS': 0, 'NOOP': 0, 'ERROR': 1, 'INPROGRESS': 0, 'PENDING': 1} + assert next(graph.wait_finish(simple.graph['uid'], 10)) == { + 'SKIPPED': 0, 'SUCCESS': 0, 'NOOP': 0, 'ERROR': 1, 'INPROGRESS': 0, 'PENDING': 1} simple.node['echo_stuff']['status'] = states.ERROR.name graph.update_graph(simple) - assert next(graph.wait_finish(simple.graph['uid'], 10)) == {'SKIPPED': 0, 'SUCCESS': 0, 'NOOP': 0, 'ERROR': 2, 'INPROGRESS': 0, 'PENDING': 0} + assert next(graph.wait_finish(simple.graph['uid'], 10)) == { + 'SKIPPED': 0, 'SUCCESS': 0, 'NOOP': 0, 'ERROR': 2, 'INPROGRESS': 0, 'PENDING': 0} diff --git a/solar/test/test_graph_filtering.py b/solar/test/test_graph_filtering.py index 0fc494bc..df24ca51 100644 --- a/solar/test/test_graph_filtering.py +++ b/solar/test/test_graph_filtering.py @@ -42,6 +42,7 @@ def dg_ex1(): def test_end_at(dg_ex1, end_nodes, visited): assert set(filters.end_at(dg_ex1, end_nodes)) == visited + @mark.parametrize("start_nodes,visited", [ (['n3'], {'n3'}), (['n1'], {'n1', 'n2', 'n4'}), @@ -50,6 +51,7 @@ def test_end_at(dg_ex1, end_nodes, visited): def test_start_from(dg_ex1, start_nodes, visited): assert set(filters.start_from(dg_ex1, start_nodes)) == visited + @fixture def dg_ex2(): dg = nx.DiGraph() @@ -68,11 +70,13 @@ def riak_plan(): def test_riak_start_node1(riak_plan): - assert filters.start_from(riak_plan, ['node1.run']) == {'node1.run', 'hosts_file1.run', 'riak_service1.run'} + assert filters.start_from(riak_plan, ['node1.run']) == { + 'node1.run', 'hosts_file1.run', 'riak_service1.run'} def test_riak_end_hosts_file1(riak_plan): - assert filters.end_at(riak_plan, ['hosts_file1.run']) == {'node1.run', 'hosts_file1.run'} + assert filters.end_at(riak_plan, ['hosts_file1.run']) == { + 'node1.run', 'hosts_file1.run'} def test_start_at_two_nodes(riak_plan): @@ -83,7 +87,8 @@ def test_start_at_two_nodes(riak_plan): def test_initial_from_node1_traverse(riak_plan): filters.filter(riak_plan, start=['node1.run']) - pending = {n for n in riak_plan if riak_plan.node[n]['status'] == states.PENDING.name} + pending = {n for n in riak_plan if riak_plan.node[ + n]['status'] == states.PENDING.name} assert pending == {'hosts_file1.run', 'riak_service1.run', 'node1.run'} @@ -92,7 +97,8 @@ def test_second_from_node2_with_node1_walked(riak_plan): for n in success: riak_plan.node[n]['status'] = states.SUCCESS.name filters.filter(riak_plan, start=['node2.run']) - pending = {n for n in riak_plan if riak_plan.node[n]['status'] == states.PENDING.name} + pending = {n for n in riak_plan if riak_plan.node[ + n]['status'] == states.PENDING.name} assert pending == {'hosts_file2.run', 'riak_service2.run', 'node2.run', 'riak_service2.join'} @@ -102,6 +108,7 @@ def test_end_joins(riak_plan): riak_plan, start=['node1.run', 'node2.run', 'node3.run'], end=['riak_service2.join', 'riak_service3.join']) - skipped = {n for n in riak_plan if riak_plan.node[n]['status'] == states.SKIPPED.name} + skipped = {n for n in riak_plan if riak_plan.node[ + n]['status'] == states.SKIPPED.name} assert skipped == {'riak_service1.commit'} diff --git a/solar/test/test_limits.py b/solar/test/test_limits.py index b4668e91..130220fc 100644 --- a/solar/test/test_limits.py +++ b/solar/test/test_limits.py @@ -46,8 +46,8 @@ def test_type_limit_rule(dg): def test_items_rule(dg): - assert limits.items_rule(dg, ['1']*99, '2') - assert limits.items_rule(dg, ['1']*99, '2', limit=10) == False + assert limits.items_rule(dg, ['1'] * 99, '2') + assert limits.items_rule(dg, ['1'] * 99, '2', limit=10) == False @fixture diff --git a/solar/test/test_operations_with_tags.py b/solar/test/test_operations_with_tags.py index ea9bfe5c..67a25e39 100644 --- a/solar/test/test_operations_with_tags.py +++ b/solar/test/test_operations_with_tags.py @@ -23,13 +23,13 @@ from solar.dblayer.model import ModelMeta def tagged_resources(): tags = ['n1', 'n2', 'n3'] t1 = Resource.from_dict('t1', - {'name': 't1', 'tags': tags, 'base_path': 'x'}) + {'name': 't1', 'tags': tags, 'base_path': 'x'}) t1.save_lazy() t2 = Resource.from_dict('t2', - {'name': 't2', 'tags': tags, 'base_path': 'x'}) + {'name': 't2', 'tags': tags, 'base_path': 'x'}) t2.save_lazy() t3 = Resource.from_dict('t3', - {'name': 't3', 'tags': tags, 'base_path': 'x'}) + {'name': 't3', 'tags': tags, 'base_path': 'x'}) t3.save_lazy() ModelMeta.save_all_lazy() return [t1, t2, t3] diff --git a/solar/test/test_resource.py b/solar/test/test_resource.py index 9252c8a0..fb1b81be 100644 --- a/solar/test/test_resource.py +++ b/solar/test/test_resource.py @@ -19,6 +19,7 @@ from solar.core import signals class TestResource(base.BaseResourceTest): + def test_resource_args(self): sample_meta_dir = self.make_resource_meta(""" id: sample diff --git a/solar/test/test_signals.py b/solar/test/test_signals.py index e2504b78..bd0ac888 100644 --- a/solar/test/test_signals.py +++ b/solar/test/test_signals.py @@ -20,6 +20,7 @@ import pytest class TestBaseInput(base.BaseResourceTest): + def test_no_self_connection(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -40,7 +41,6 @@ input: 'Trying to connect value-.* to itself'): xs.connect(sample, sample, {'value'}) - def test_input_dict_type(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -83,7 +83,7 @@ input: sample1.args['values'], {'a': 3} ) - #self.assertEqual( + # self.assertEqual( # sample2.args['values'], # {'a': 2} #) @@ -138,11 +138,11 @@ input: xs.connect(sample_port, sample) self.assertEqual(sample.args['ip'], sample_ip.args['ip']) self.assertEqual(sample.args['port'], sample_port.args['port']) - #self.assertEqual( + # self.assertEqual( # sample.args['ip'].emitter, # sample_ip.args['ip'] #) - #self.assertEqual( + # self.assertEqual( # sample.args['port'].emitter, # sample_port.args['port'] #) @@ -171,7 +171,7 @@ input: xs.connect(sample1, sample) self.assertEqual(sample1.args['ip'], sample.args['ip']) #self.assertEqual(len(list(sample1.args['ip'].receivers)), 1) - #self.assertEqual( + # self.assertEqual( # sample.args['ip'].emitter, # sample1.args['ip'] #) @@ -211,6 +211,7 @@ input: class TestListInput(base.BaseResourceTest): + def test_list_input_single(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -249,7 +250,7 @@ input: sample1.args['ip'], ] ) - #self.assertListEqual( + # self.assertListEqual( # [(e['emitter_attached_to'], e['emitter']) for e in list_input_single.args['ips']], # [(sample1.args['ip'].attached_to.name, 'ip')] #) @@ -263,7 +264,7 @@ input: sample2.args['ip'], ] ) - #self.assertListEqual( + # self.assertListEqual( # [(e['emitter_attached_to'], e['emitter']) for e in list_input_single.args['ips']], # [(sample1.args['ip'].attached_to.name, 'ip'), # (sample2.args['ip'].attached_to.name, 'ip')] @@ -289,7 +290,7 @@ input: sample1.args['ip'], ] ) - #self.assertListEqual( + # self.assertListEqual( # [(e['emitter_attached_to'], e['emitter']) for e in list_input_single.args['ips']], # [(sample1.args['ip'].attached_to.name, 'ip')] #) @@ -330,7 +331,8 @@ input: 'list-input-multi', list_input_multi_meta_dir, args={'ips': [], 'ports': []} ) - xs.connect(sample1, list_input_multi, mapping={'ip': 'ips', 'port': 'ports'}) + xs.connect(sample1, list_input_multi, mapping={ + 'ip': 'ips', 'port': 'ports'}) self.assertItemsEqual( #[ip['value'] for ip in list_input_multi.args['ips']], list_input_multi.args['ips'], @@ -342,7 +344,8 @@ input: [sample1.args['port']] ) - xs.connect(sample2, list_input_multi, mapping={'ip': 'ips', 'port': 'ports'}) + xs.connect(sample2, list_input_multi, mapping={ + 'ip': 'ips', 'port': 'ports'}) self.assertItemsEqual( #[ip['value'] for ip in list_input_multi.args['ips']], list_input_multi.args['ips'], @@ -351,7 +354,7 @@ input: sample2.args['ip'], ] ) - #self.assertListEqual( + # self.assertListEqual( # [(e['emitter_attached_to'], e['emitter']) for e in list_input_multi.args['ips']], # [(sample1.args['ip'].attached_to.name, 'ip'), # (sample2.args['ip'].attached_to.name, 'ip')] @@ -364,7 +367,7 @@ input: sample2.args['port'], ] ) - #self.assertListEqual( + # self.assertListEqual( # [(e['emitter_attached_to'], e['emitter']) for e in list_input_multi.args['ports']], # [(sample1.args['port'].attached_to.name, 'port'), # (sample2.args['port'].attached_to.name, 'port')] @@ -469,6 +472,7 @@ input: class TestHashInput(base.BaseResourceTest): + @pytest.mark.xfail(reason="Connect should raise an error if already connected") def test_hash_input_basic(self): sample_meta_dir = self.make_resource_meta(""" @@ -504,7 +508,8 @@ input: receiver = self.create_resource( 'receiver', receiver_meta_dir ) - xs.connect(sample1, receiver, mapping={'ip': 'server:ip', 'port': 'server:port'}) + xs.connect(sample1, receiver, mapping={ + 'ip': 'server:ip', 'port': 'server:port'}) self.assertDictEqual( {'ip': sample1.args['ip'], 'port': sample1.args['port']}, receiver.args['server'], @@ -521,7 +526,8 @@ input: {'ip': sample2.args['ip'], 'port': sample1.args['port']}, receiver.args['server'], ) - xs.connect(sample3, receiver, mapping={'ip': 'server:ip', 'port': 'server:port'}) + xs.connect(sample3, receiver, mapping={ + 'ip': 'server:ip', 'port': 'server:port'}) self.assertDictEqual( {'ip': sample3.args['ip'], 'port': sample3.args['port']}, receiver.args['server'], @@ -594,7 +600,8 @@ input: receiver = self.create_resource( 'receiver', receiver_meta_dir ) - xs.connect(sample1, receiver, mapping={'ip': 'server:ip', 'port': 'server:port'}) + xs.connect(sample1, receiver, mapping={ + 'ip': 'server:ip', 'port': 'server:port'}) self.assertItemsEqual( [{'ip': sample1.args['ip'], 'port': sample1.args['port']}], receiver.args['server'], @@ -602,7 +609,8 @@ input: sample2 = self.create_resource( 'sample2', sample_meta_dir, args={'ip': '10.0.0.2', 'port': 5001} ) - xs.connect(sample2, receiver, mapping={'ip': 'server:ip', 'port': 'server:port'}) + xs.connect(sample2, receiver, mapping={ + 'ip': 'server:ip', 'port': 'server:port'}) self.assertItemsEqual( [{'ip': sample1.args['ip'], 'port': sample1.args['port']}, {'ip': sample2.args['ip'], 'port': sample2.args['port']}], @@ -692,7 +700,8 @@ input: sample3 = self.create_resource( 'sample3', sample_meta_dir, args={'ip': '10.0.0.3', 'port': 5002} ) - sample3.connect(receiver, mapping={'ip': 'server:ip', 'port': 'server:port'}) + sample3.connect(receiver, mapping={ + 'ip': 'server:ip', 'port': 'server:port'}) self.assertItemsEqual( [{'ip': sample1.args['ip'], 'port': sample2.args['port']}, {'ip': sample3.args['ip'], 'port': sample3.args['port']}], diff --git a/solar/test/test_system_log_api.py b/solar/test/test_system_log_api.py index 2c7db519..5d897fbc 100644 --- a/solar/test/test_system_log_api.py +++ b/solar/test/test_system_log_api.py @@ -30,8 +30,8 @@ def test_revert_update(): commit = {'a': '10'} previous = {'a': '9'} res = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', 'base_path': 'x', + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res.save() action = 'update' res.inputs['a'] = '9' @@ -40,7 +40,7 @@ def test_revert_update(): assert resource_obj.args == previous log = data.SL() - logitem =change.create_logitem( + logitem = change.create_logitem( res.name, action, change.create_diff(commit, previous), [], base_path=res.base_path) log.append(logitem) @@ -56,23 +56,23 @@ def test_revert_update(): def test_revert_update_connected(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() res2 = DBResource.from_dict('test2', - {'name': 'test2', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test2', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res2.inputs['a'] = '' res2.save_lazy() res3 = DBResource.from_dict('test3', - {'name': 'test3', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test3', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res3.inputs['a'] = '' res3.save_lazy() @@ -113,15 +113,15 @@ def test_revert_update_connected(): def test_revert_removal(): res = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res.inputs['a'] = '9' res.save_lazy() commited = CommitedResource.from_dict('test1', - {'inputs': {'a': '9'}, - 'state': 'operational'}) + {'inputs': {'a': '9'}, + 'state': 'operational'}) commited.save_lazy() resource_obj = resource.load(res.name) @@ -138,7 +138,8 @@ def test_revert_removal(): assert DBResource.bucket.get('test1').siblings == [] with mock.patch.object(resource, 'read_meta') as mread: - mread.return_value = {'input': {'a': {'schema': 'str!'}}, 'id': 'mocked'} + mread.return_value = { + 'input': {'a': {'schema': 'str!'}}, 'id': 'mocked'} change.revert(changes[0].uid) ModelMeta.save_all_lazy() assert len(DBResource.bucket.get('test1').siblings) == 1 @@ -183,9 +184,9 @@ def test_revert_removed_child(): def test_revert_create(): res = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res.inputs['a'] = '9' res.save_lazy() ModelMeta.save_all_lazy() @@ -210,16 +211,16 @@ def test_revert_create(): def test_discard_all_pending_changes_resources_created(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() res2 = DBResource.from_dict('test2', - {'name': 'test2', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test2', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res2.inputs['a'] = '0' res2.save_lazy() ModelMeta.save_all_lazy() @@ -235,16 +236,16 @@ def test_discard_all_pending_changes_resources_created(): def test_discard_connection(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() res2 = DBResource.from_dict('test2', - {'name': 'test2', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test2', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res2.inputs['a'] = '0' res2.save_lazy() ModelMeta.save_all_lazy() @@ -266,9 +267,9 @@ def test_discard_connection(): def test_discard_removed(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() ModelMeta.save_all_lazy() @@ -288,9 +289,9 @@ def test_discard_removed(): def test_discard_update(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', 'base_path': 'x', + 'state': RESOURCE_STATE.created.name, + 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() ModelMeta.save_all_lazy() diff --git a/solar/test/test_system_log_details.py b/solar/test/test_system_log_details.py index 20fc748b..dfff8a24 100644 --- a/solar/test/test_system_log_details.py +++ b/solar/test/test_system_log_details.py @@ -22,10 +22,11 @@ def host_diff(): return [ [u'add', u'', [ [u'ip', u'10.0.0.3'], - [u'hosts_names', ['riak_server1.solar', 'riak_server2.solar', 'riak_server3.solar']], + [u'hosts_names', ['riak_server1.solar', + 'riak_server2.solar', 'riak_server3.solar']], [u'ssh_user', u'vagrant'], [u'ssh_key', u'/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key'], - ]]] + ]]] def test_details_for_add(host_diff): @@ -39,8 +40,10 @@ def test_details_for_add(host_diff): def list_change(): return [[u'change', [u'configs_ports', 0, u'value', 0, u'value'], [18098, 88888]]] + def test_list_details_for_change(list_change): - assert data.details(list_change) == ['-+ configs_ports:[0] : 18098 >> 88888'] + assert data.details(list_change) == [ + '-+ configs_ports:[0] : 18098 >> 88888'] @fixture diff --git a/solar/test/test_traversal.py b/solar/test/test_traversal.py index 13595e76..ba609a09 100644 --- a/solar/test/test_traversal.py +++ b/solar/test/test_traversal.py @@ -17,6 +17,7 @@ from pytest import fixture from solar.orchestration.traversal import traverse + @fixture def tasks(): return [ @@ -26,6 +27,7 @@ def tasks(): {'id': 't4', 'status': 'PENDING'}, {'id': 't5', 'status': 'PENDING'}] + @fixture def dg(tasks): ex = nx.DiGraph() @@ -61,6 +63,7 @@ def test_nothing_will_be_walked_if_parent_is_skipped(dg): assert set(traverse(dg)) == set() + def test_node_will_be_walked_if_parent_is_noop(dg): dg.add_path(['t1', 't2', 't3', 't4', 't5']) dg.node['t1']['status'] = 'NOOP' diff --git a/solar/test/test_validation.py b/solar/test/test_validation.py index 3bbf0b4a..acf67b9c 100644 --- a/solar/test/test_validation.py +++ b/solar/test/test_validation.py @@ -19,6 +19,7 @@ from solar.core import validation as sv class TestInputValidation(base.BaseResourceTest): + def test_input_str_type(self): sample_meta_dir = self.make_resource_meta(""" id: sample diff --git a/solar/test/test_virtual_resource.py b/solar/test/test_virtual_resource.py index 41cf0067..8e807517 100644 --- a/solar/test/test_virtual_resource.py +++ b/solar/test/test_virtual_resource.py @@ -36,6 +36,7 @@ def good_events(): ''' return yaml.load(StringIO(events)) + @pytest.fixture def bad_event_type(): events = ''' @@ -46,12 +47,14 @@ def bad_event_type(): ''' return yaml.load(StringIO(events)) + def test_create_path_does_not_exists(): with pytest.raises(Exception) as excinfo: vr.create('node1', '/path/does/not/exists') err = 'Base resource does not exist: /path/does/not/exists' assert str(excinfo.value) == err + def test_create_resource(): node_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -60,6 +63,7 @@ def test_create_resource(): assert len(resources) == 1 assert resources[0].name == 'node1' + def test_create_virtual_resource(tmpdir): base_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -73,6 +77,7 @@ def test_create_virtual_resource(tmpdir): resources = vr.create('nodes', str(vr_file)) assert len(resources) == 2 + def test_create_virtual_resource_with_list(tmpdir): base_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -110,16 +115,18 @@ def test_update(tmpdir): vr.create('updates', str(update_file)) assert resources[0].args['ip'] == '10.0.0.4' + def test_parse_events(good_events): - events =[Dep(parent='service1', parent_action='run', - child='config1', child_action='run', - state='success'), - React(parent='config1', parent_action='run', - child='service1', child_action='apply_config', - state='success')] + events = [Dep(parent='service1', parent_action='run', + child='config1', child_action='run', + state='success'), + React(parent='config1', parent_action='run', + child='service1', child_action='apply_config', + state='success')] parsed = vr.parse_events(good_events) assert events == parsed + def test_parse_bad_event(bad_event_type): with pytest.raises(Exception) as execinfo: vr.parse_events(bad_event_type) @@ -128,43 +135,47 @@ def test_parse_bad_event(bad_event_type): def test_add_connections(mocker, resources): - mocked_signals = mocker.patch('solar.core.resource.resource.Resource.connect_with_events') + mocked_signals = mocker.patch( + 'solar.core.resource.resource.Resource.connect_with_events') args = {'ip': 'node1::ip', 'servers': ['node1::ip', 'node2::ip'], 'alias': 'ser1' - } + } vr.update_inputs('service1', args) assert mocked_signals.call_count == 2 def test_add_list_values(mocker, resources): - mocked_signals = mocker.patch('solar.core.resource.resource.Resource.connect_with_events') + mocked_signals = mocker.patch( + 'solar.core.resource.resource.Resource.connect_with_events') args = {'ip': 'node1::ip', 'servers': ['server1', 'server2'], 'alias': 'ser1' - } + } vr.update_inputs('service1', args) assert mocked_signals.call_count == 1 def test_parse_connection(): correct_connection = {'child_input': 'ip', - 'parent' : 'node1', - 'parent_input': 'ip', - 'events' : None - } + 'parent': 'node1', + 'parent_input': 'ip', + 'events': None + } connection = vr.parse_connection('ip', 'node1::ip') assert correct_connection == connection + def test_parse_connection_disable_events(): correct_connection = {'child_input': 'ip', - 'parent' : 'node1', - 'parent_input': 'ip', - 'events' : False - } + 'parent': 'node1', + 'parent_input': 'ip', + 'events': False + } connection = vr.parse_connection('ip', 'node1::ip::NO_EVENTS') assert correct_connection == connection + def test_setting_location(tmpdir): # XXX: make helper for it base_path = os.path.join( diff --git a/solar/utils.py b/solar/utils.py index f67a86ad..facc3506 100644 --- a/solar/utils.py +++ b/solar/utils.py @@ -12,17 +12,17 @@ # License for the specific language governing permissions and limitations # under the License. +import glob import io import json -import glob -import yaml import logging import os - -from uuid import uuid4 +import uuid +import yaml from jinja2 import Environment + logger = logging.getLogger(__name__) @@ -80,7 +80,7 @@ def load_by_mask(mask): def generate_uuid(): - return str(uuid4()) + return str(uuid.uuid4()) def render_template(template_path, **params): @@ -134,5 +134,5 @@ def solar_map(funct, args, **kwargs): def get_local(): - from threading import local - return local + import threading + return threading.local diff --git a/tox.ini b/tox.ini index 4922b051..75fefd23 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ deps = -r{toxinidir}/test-requirements.txt commands = ostestr --serial [testenv:pep8] -deps = hacking==0.7 +deps = hacking==0.9.6 usedevelop = False commands = flake8 {posargs:solar} @@ -27,10 +27,7 @@ envdir = devenv usedevelop = True [flake8] -# NOTE(eli): H304 is "No relative imports" error, relative -# imports are required for extensions which can be moved -# from nailgun directory to different place -ignore = H234,H302,H802,H304 +ignore = E731 exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs show-pep8 = True show-source = True From f20c3f1c6c8bc14554da470dde37e49bcfdfb5c3 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 26 Nov 2015 12:56:59 +0100 Subject: [PATCH 2/5] Orchestration adjusted to OpenStack Style Guidelines --- solar/orchestration/executor.py | 6 ++++-- solar/orchestration/filters.py | 10 ++++++---- solar/orchestration/graph.py | 25 +++++++++++-------------- solar/orchestration/limits.py | 4 +++- solar/orchestration/tasks.py | 17 +++++++++-------- solar/orchestration/utils.py | 3 ++- 6 files changed, 35 insertions(+), 30 deletions(-) diff --git a/solar/orchestration/executor.py b/solar/orchestration/executor.py index d4250dce..a1cfeb3a 100644 --- a/solar/orchestration/executor.py +++ b/solar/orchestration/executor.py @@ -14,9 +14,10 @@ import time -from solar.orchestration.runner import app from celery import group +from solar.orchestration.runner import app + def celery_executor(dg, tasks, control_tasks=()): to_execute = [] @@ -28,7 +29,8 @@ def celery_executor(dg, tasks, control_tasks=()): task_id = '{}:{}'.format(dg.graph['uid'], task_name) task = app.tasks[dg.node[task_name]['type']] - if all_success(dg, dg.predecessors(task_name)) or task_name in control_tasks: + all_ok = all_success(dg, dg.predecessors(task_name)) + if all_ok or task_name in control_tasks: dg.node[task_name]['status'] = 'INPROGRESS' dg.node[task_name]['start_time'] = time.time() for t in generate_task(task, dg.node[task_name], task_id): diff --git a/solar/orchestration/filters.py b/solar/orchestration/filters.py index ea1cf793..05289934 100644 --- a/solar/orchestration/filters.py +++ b/solar/orchestration/filters.py @@ -14,7 +14,8 @@ import networkx as nx -from .traversal import VISITED, states +from solar.orchestration.traversal import states +from solar.orchestration.traversal import VISITED def get_dfs_postorder_subgraph(dg, nodes): @@ -26,6 +27,7 @@ def get_dfs_postorder_subgraph(dg, nodes): def end_at(dg, nodes): """Returns subgraph that will guarantee that predecessors are visited + dg - directed graph nodes - iterable with node names """ @@ -33,8 +35,7 @@ def end_at(dg, nodes): def start_from(dg, start_nodes): - """Guarantee that all paths starting from specific *nodes* will be visited - """ + """Ensures that all paths starting from specific *nodes* will be visited""" visited = {n for n in dg if dg.node[n].get('status') in VISITED} # sorting nodes in topological order will guarantee that all predecessors @@ -71,7 +72,8 @@ def validate(dg, start_nodes, end_nodes, err_msgs): def filter(dg, start=None, end=None, tasks=(), skip_with=states.SKIPPED.name): - """ + """Filters a graph + TODO(dshulyak) skip_with should also support NOOP, which will instead of blocking task, and its successors, should mark task as visited diff --git a/solar/orchestration/graph.py b/solar/orchestration/graph.py index 50670e30..59435c30 100644 --- a/solar/orchestration/graph.py +++ b/solar/orchestration/graph.py @@ -12,19 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. -import uuid +from collections import Counter import time +import uuid import networkx as nx -from solar import utils -from .traversal import states -from solar import errors - -from collections import Counter - -from solar.dblayer.solar_models import Task from solar.dblayer.model import clear_cache +from solar.dblayer.solar_models import Task +from solar import errors +from solar import utils + +from solar.orchestration.traversal import states def save_graph(graph): @@ -77,8 +76,7 @@ get_plan = get_graph def parse_plan(plan_path): - """ parses yaml definition and returns graph - """ + """parses yaml definition and returns graph""" plan = utils.yaml_load(plan_path) dg = nx.MultiDiGraph() dg.graph['name'] = plan['name'] @@ -123,8 +121,6 @@ def show(uid): def create_plan(plan_path, save=True): - """ - """ dg = parse_plan(plan_path) return create_plan_from_graph(dg, save=save) @@ -163,8 +159,9 @@ def report_topo(uid): def wait_finish(uid, timeout): - """Wait finish will periodically load graph and check if there is no - PENDING or INPROGRESS + """Check if graph is finished + + Will return when no PENDING or INPROGRESS otherwise yields summary """ start_time = time.time() diff --git a/solar/orchestration/limits.py b/solar/orchestration/limits.py index 6c19f340..060373ad 100644 --- a/solar/orchestration/limits.py +++ b/solar/orchestration/limits.py @@ -47,7 +47,9 @@ def get_default_chain(dg, inprogress, added): def type_based_rule(dg, inprogress, item): - """condition will be specified like: + """Checks type based rules + + condition should be specified like: type_limit: 2 """ _type = dg.node[item].get('resource_type') diff --git a/solar/orchestration/tasks.py b/solar/orchestration/tasks.py index 600734a7..e392036e 100644 --- a/solar/orchestration/tasks.py +++ b/solar/orchestration/tasks.py @@ -14,21 +14,22 @@ from functools import partial import subprocess -import time from celery.app import task -from celery.signals import task_prerun, task_postrun +from celery.signals import task_postrun +from celery.signals import task_prerun -from solar.orchestration import graph from solar.core import actions from solar.core import resource -from solar.system_log.tasks import commit_logitem, error_logitem +from solar.dblayer import ModelMeta +from solar.orchestration import executor +from solar.orchestration import graph +from solar.orchestration import limits from solar.orchestration.runner import app from solar.orchestration.traversal import traverse -from solar.orchestration import limits -from solar.orchestration import executor -from solar.dblayer import ModelMeta - +from solar.system_log.tasks import commit_logitem +from solar.system_log.tasks import error_logitem +import time __all__ = ['solar_resource', 'cmd', 'sleep', 'error', 'fault_tolerance', 'schedule_start', 'schedule_next'] diff --git a/solar/orchestration/utils.py b/solar/orchestration/utils.py index 375b5969..c915da8c 100644 --- a/solar/orchestration/utils.py +++ b/solar/orchestration/utils.py @@ -18,7 +18,8 @@ import networkx as nx def write_graph(plan): - """ + """Writes graph to dot then to svg + :param plan: networkx Graph object """ colors = { From a62e603ca56212fc59ad18f3f04666a3bc61833d Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 26 Nov 2015 13:01:38 +0100 Subject: [PATCH 3/5] Adjusted events to OpenStack Style Guidelines --- solar/events/api.py | 11 +++++++---- solar/events/controls.py | 9 +++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/solar/events/api.py b/solar/events/api.py index a814cb0f..7d2235ac 100644 --- a/solar/events/api.py +++ b/solar/events/api.py @@ -18,9 +18,9 @@ __all__ = ['add_dep', 'add_react', 'Dep', 'React', 'add_event'] import networkx as nx from solar.core.log import log -from solar.events.controls import Dep, React, StateChange - from solar.dblayer.solar_models import Resource +from solar.events.controls import Dep +from solar.events.controls import React def create_event(event_dict): @@ -99,6 +99,7 @@ def all_events(resource): def bft_events_graph(start): """Builds graph of events traversing events in breadth-first order + This graph doesnt necessary reflect deployment order, it is used to show dependencies between resources """ @@ -127,7 +128,8 @@ def bft_events_graph(start): def build_edges(changes_graph, events): - """ + """Builds graph edges + :param changes_graph: nx.DiGraph object with actions to be executed :param events: {res: [controls.Event objects]} """ @@ -149,7 +151,8 @@ def build_edges(changes_graph, events): log.debug('No outgoing events based on %s', event_name) if event_name not in visited: - for parent, child, data in events_graph.edges(event_name, data=True): + for parent, child, data in events_graph.edges(event_name, + data=True): succ_ev = data['event'] succ_ev.insert(stack, changes_graph) visited.add(event_name) diff --git a/solar/events/controls.py b/solar/events/controls.py index 2072fcc6..3a96daea 100644 --- a/solar/events/controls.py +++ b/solar/events/controls.py @@ -23,16 +23,17 @@ if dependent resource isnt changed. depends_on: - parent:run -> ok -> dependent:run -*react_on* - relationship which will guarantee that action on dependent resource -will be executed if parent action is going to be executed. This control will -trigger action even if no changes noticed on dependent resource. +*react_on* - relationship which will guarantee that action on dependent +resource will be executed if parent action is going to be executed. +This control will trigger action even +if no changes noticed on dependent resource. react_on: - parent:update -> ok -> dependent:update """ -from solar.dblayer.solar_models import Resource from solar.dblayer.model import DBLayerNotFound +from solar.dblayer.solar_models import Resource class Event(object): From 67e1f21481e8cd8c0be1dc8af6f8aeb47575feaa Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 26 Nov 2015 13:15:38 +0100 Subject: [PATCH 4/5] system_log adjusted to OpenStack Style Guidelines --- solar/system_log/change.py | 50 ++++++++++++++++++---------------- solar/system_log/operations.py | 8 ++++-- solar/system_log/tasks.py | 3 +- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/solar/system_log/change.py b/solar/system_log/change.py index 46107fdc..7779fdba 100644 --- a/solar/system_log/change.py +++ b/solar/system_log/change.py @@ -15,19 +15,19 @@ import dictdiffer import networkx as nx -from solar.system_log import data from solar.core.log import log -from solar.core import signals from solar.core import resource +from solar.core.resource.resource import RESOURCE_STATE +from solar.core import signals +from solar.dblayer.solar_models import CommitedResource +from solar.dblayer.solar_models import LogItem +from solar.dblayer.solar_models import StrInt +from solar.events import api as evapi +from solar.orchestration import graph +from solar.system_log import data from solar import utils -from solar.orchestration import graph -from solar.events import api as evapi -from .consts import CHANGES -from solar.core.resource.resource import RESOURCE_STATE -from solar.errors import CannotFindID - -from solar.dblayer.solar_models import Resource, LogItem, CommitedResource, StrInt +from solar.core.consts import CHANGES def guess_action(from_, to): @@ -135,7 +135,9 @@ def parameters(res, action, data): def _get_args_to_update(args, connections): - """For each resource we can update only args that are not provided + """Returns args to update + + For each resource we can update only args that are not provided by connections """ inherited = [i[3].split(':')[0] for i in connections] @@ -146,7 +148,8 @@ def _get_args_to_update(args, connections): def revert_uids(uids): - """ + """Reverts uids + :param uids: iterable not generator """ items = LogItem.multi_get(uids) @@ -165,15 +168,15 @@ def revert_uids(uids): def _revert_remove(logitem): - """Resource should be created with all previous connections - """ + """Resource should be created with all previous connections""" commited = CommitedResource.get(logitem.resource) args = dictdiffer.revert(logitem.diff, commited.inputs) connections = dictdiffer.revert( logitem.connections_diff, sorted(commited.connections)) resource.Resource(logitem.resource, logitem.base_path, - args=_get_args_to_update(args, connections), tags=commited.tags) + args=_get_args_to_update(args, connections), + tags=commited.tags) for emitter, emitter_input, receiver, receiver_input in connections: emmiter_obj = resource.load(emitter) receiver_obj = resource.load(receiver) @@ -181,7 +184,7 @@ def _revert_remove(logitem): emitter_input: receiver_input}) -def _update_inputs_connections(res_obj, args, old_connections, new_connections): +def _update_inputs_connections(res_obj, args, old_connections, new_connections): # NOQA removed = [] for item in old_connections: @@ -204,7 +207,8 @@ def _update_inputs_connections(res_obj, args, old_connections, new_connections): emmiter_obj.connect(receiver_obj, {emitter_input: receiver_input}) if removed or added: - # TODO without save we will get error that some values can not be updated + # TODO without save we will get error + # that some values can not be updated # even if connection was removed receiver_obj.db_obj.save() @@ -212,8 +216,7 @@ def _update_inputs_connections(res_obj, args, old_connections, new_connections): def _revert_update(logitem): - """Revert of update should update inputs and connections - """ + """Revert of update should update inputs and connections""" res_obj = resource.load(logitem.resource) commited = res_obj.load_commited() @@ -222,7 +225,8 @@ def _revert_update(logitem): args = dictdiffer.revert(logitem.diff, commited.inputs) _update_inputs_connections( - res_obj, _get_args_to_update(args, connections), commited.connections, connections) + res_obj, _get_args_to_update(args, connections), + commited.connections, connections) def _revert_run(logitem): @@ -247,7 +251,8 @@ def _discard_update(item): args = dictdiffer.revert(item.diff, resource_obj.args) _update_inputs_connections( - resource_obj, _get_args_to_update(args, new_connections), old_connections, new_connections) + resource_obj, _get_args_to_update(args, new_connections), + old_connections, new_connections) def _discard_run(item): @@ -279,9 +284,8 @@ def discard_all(): def commit_all(): - """Helper mainly for ease of testing - """ - from .operations import move_to_commited + """Helper mainly for ease of testing""" + from solar.system_log.operations import move_to_commited for item in data.SL(): move_to_commited(item.log_action) diff --git a/solar/system_log/operations.py b/solar/system_log/operations.py index 2eb9f42f..759886c4 100644 --- a/solar/system_log/operations.py +++ b/solar/system_log/operations.py @@ -12,11 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -from solar.system_log import data -from solar.dblayer.solar_models import CommitedResource from dictdiffer import patch + from solar.core.resource import resource -from .consts import CHANGES +from solar.dblayer.solar_models import CommitedResource +from solar.system_log import data + +from solar.system_log.consts import CHANGES def set_error(log_action, *args, **kwargs): diff --git a/solar/system_log/tasks.py b/solar/system_log/tasks.py index 9961837c..1c53f1b1 100644 --- a/solar/system_log/tasks.py +++ b/solar/system_log/tasks.py @@ -13,7 +13,8 @@ # under the License. from solar.orchestration.runner import app -from solar.system_log.operations import set_error, move_to_commited +from solar.system_log.operations import move_to_commited +from solar.system_log.operations import set_error __all__ = ['error_logitem', 'commit_logitem'] From ae112eeef3d53d4d5442bbe238d503a5c809c621 Mon Sep 17 00:00:00 2001 From: Jedrzej Nowak Date: Thu, 26 Nov 2015 14:09:54 +0100 Subject: [PATCH 5/5] Adjusted tests code to OpenStack Style Guidelines --- solar/test/base.py | 7 +- solar/test/conftest.py | 15 +- solar/test/test_celery_executor.py | 5 +- solar/test/test_diff_generation.py | 60 +-- solar/test/test_events.py | 56 +-- solar/test/test_graph_api.py | 28 +- solar/test/test_graph_filtering.py | 46 ++- solar/test/test_limits.py | 42 +- solar/test/test_operations_with_tags.py | 2 +- solar/test/test_resource.py | 32 +- solar/test/test_signals.py | 501 ++++++++++-------------- solar/test/test_system_log_api.py | 115 ++++-- solar/test/test_validation.py | 4 +- solar/test/test_virtual_resource.py | 3 +- 14 files changed, 447 insertions(+), 469 deletions(-) diff --git a/solar/test/base.py b/solar/test/base.py index 597fb14c..481f2443 100644 --- a/solar/test/base.py +++ b/solar/test/base.py @@ -15,12 +15,13 @@ import os import shutil import tempfile -import unittest -import yaml import time +import unittest + +import yaml from solar.core.resource import virtual_resource as vr -from solar.dblayer.model import Model, Replacer, ModelMeta, get_bucket +from solar.dblayer.model import Model def patched_get_bucket_name(cls): diff --git a/solar/test/conftest.py b/solar/test/conftest.py index fff7d599..d1c12a1b 100644 --- a/solar/test/conftest.py +++ b/solar/test/conftest.py @@ -14,11 +14,13 @@ import os import time -from solar.core.resource import Resource -from solar.dblayer.model import Model, ModelMeta, get_bucket - import pytest +from solar.core.resource import Resource +from solar.dblayer.model import Model +from solar.dblayer.model import ModelMeta +from solar.dblayer.model import get_bucket + def patched_get_bucket_name(cls): return cls.__name__ + str(time.time()) @@ -49,13 +51,6 @@ def setup(request): model.bucket = get_bucket(None, model, ModelMeta) -@pytest.fixture(autouse=True) -def setup(request): - - for model in ModelMeta._defined_models: - model.bucket = get_bucket(None, model, ModelMeta) - - def pytest_runtest_teardown(item, nextitem): ModelMeta.session_end(result=True) return nextitem diff --git a/solar/test/test_celery_executor.py b/solar/test/test_celery_executor.py index 6b58031a..f96d13dc 100644 --- a/solar/test/test_celery_executor.py +++ b/solar/test/test_celery_executor.py @@ -13,8 +13,8 @@ # under the License. import networkx as nx -from pytest import fixture from mock import patch +from pytest import fixture from solar.orchestration import executor @@ -29,7 +29,6 @@ def dg(): @patch.object(executor, 'app') def test_celery_executor(mapp, dg): - """Just check that it doesnt fail for now. - """ + """Just check that it doesnt fail for now.""" assert executor.celery_executor(dg, ['t1']) assert dg.node['t1']['status'] == 'INPROGRESS' diff --git a/solar/test/test_diff_generation.py b/solar/test/test_diff_generation.py index 6f09204f..cb8c710a 100644 --- a/solar/test/test_diff_generation.py +++ b/solar/test/test_diff_generation.py @@ -12,9 +12,9 @@ # License for the specific language governing permissions and limitations # under the License. +from dictdiffer import patch +from dictdiffer import revert from pytest import fixture -from dictdiffer import revert, patch -import networkx as nx from solar.system_log import change @@ -28,8 +28,8 @@ def staged(): 'metadata': {}, 'connections': [ ['node.1', 'res.1', ['ip', 'ip']], - ['node.1', 'res.1', ['key', 'key']]] - } + ['node.1', 'res.1', ['key', 'key']] + ]} @fixture @@ -40,8 +40,8 @@ def commited(): 'list_val': [1]}, 'metadata': {}, 'connections': [ - ['node.1', 'res.1', ['ip', 'ip']]] - } + ['node.1', 'res.1', ['ip', 'ip']] + ]} @fixture @@ -56,17 +56,24 @@ def diff_for_update(staged, commited): def test_create_diff_with_empty_commited(full_diff): # add will be executed - expected = [('add', '', [('connections', [['node.1', 'res.1', ['ip', 'ip']], ['node.1', 'res.1', ['key', 'key']]]), ('input', { - 'ip': {'value': '10.0.0.2'}, 'list_val': {'value': [1, 2]}}), ('metadata', {}), ('id', 'res.1'), ('tags', ['res', 'node.1'])])] + expected = [('add', '', + [('connections', [['node.1', 'res.1', ['ip', 'ip']], + ['node.1', 'res.1', ['key', 'key']]]), + ('input', { + 'ip': {'value': '10.0.0.2'}, + 'list_val': {'value': [1, 2]} + }), ('metadata', {}), ('id', 'res.1'), + ('tags', ['res', 'node.1'])])] assert full_diff == expected def test_create_diff_modified(diff_for_update): assert diff_for_update == [ - ('add', 'connections', - [(1, ['node.1', 'res.1', ['key', 'key']])]), - ('change', 'input.ip', ('10.0.0.2', {'value': '10.0.0.2'})), - ('change', 'input.list_val', ([1], {'value': [1, 2]}))] + ('add', 'connections', [(1, ['node.1', 'res.1', ['key', 'key']])]), + ('change', 'input.ip', + ('10.0.0.2', {'value': '10.0.0.2'})), ('change', 'input.list_val', + ([1], {'value': [1, 2]})) + ] def test_verify_patch_creates_expected(staged, diff_for_update, commited): @@ -81,20 +88,17 @@ def test_revert_update(staged, diff_for_update, commited): @fixture def resources(): - r = {'n.1': - {'uid': 'n.1', - 'args': {'ip': '10.20.0.2'}, - 'connections': [], - 'tags': []}, - 'r.1': - {'uid': 'r.1', - 'args': {'ip': '10.20.0.2'}, - 'connections': [['n.1', 'r.1', ['ip', 'ip']]], - 'tags': []}, - 'h.1': - {'uid': 'h.1', - 'args': {'ip': '10.20.0.2', - 'ips': ['10.20.0.2']}, - 'connections': [['n.1', 'h.1', ['ip', 'ip']]], - 'tags': []}} + r = {'n.1': {'uid': 'n.1', + 'args': {'ip': '10.20.0.2'}, + 'connections': [], + 'tags': []}, + 'r.1': {'uid': 'r.1', + 'args': {'ip': '10.20.0.2'}, + 'connections': [['n.1', 'r.1', ['ip', 'ip']]], + 'tags': []}, + 'h.1': {'uid': 'h.1', + 'args': {'ip': '10.20.0.2', + 'ips': ['10.20.0.2']}, + 'connections': [['n.1', 'h.1', ['ip', 'ip']]], + 'tags': []}} return r diff --git a/solar/test/test_events.py b/solar/test/test_events.py index 1ef78cb0..24136f21 100644 --- a/solar/test/test_events.py +++ b/solar/test/test_events.py @@ -16,10 +16,8 @@ import networkx as nx from pytest import fixture -from solar.events import api as evapi from solar.dblayer.solar_models import Resource - -from .base import BaseResourceTest +from solar.events import api as evapi @fixture @@ -76,9 +74,7 @@ def test_nova_api_run_after_nova(nova_deps): def test_nova_api_react_on_update(nova_deps): - """Test that nova_api:update will be called even if there is no changes - in nova_api - """ + """Test that nova_api:update will be called even if there is no changes in nova_api""" # NOQA changes_graph = nx.DiGraph() changes_graph.add_node('nova.update') evapi.build_edges(changes_graph, nova_deps) @@ -89,19 +85,25 @@ def test_nova_api_react_on_update(nova_deps): @fixture def rmq_deps(): """Example of a case when defaults are not good enough. + For example we need to run some stuff on first node before two others. """ # NOTE(dshulyak) is it possible to put this create/join logic into # puppet manifest? So that rmq_cluster.2 before joining will check if # cluster already exists? return { - 'rmq.1': [evapi.Dep('rmq.1', 'run', 'success', 'rmq_cluster.1', 'create')], - 'rmq.2': [evapi.Dep('rmq.2', 'run', 'success', 'rmq_cluster.2', 'join')], - 'rmq.3': [evapi.Dep('rmq.3', 'run', 'success', 'rmq_cluster.3', 'join')], + 'rmq.1': + [evapi.Dep('rmq.1', 'run', 'success', 'rmq_cluster.1', 'create')], + 'rmq.2': + [evapi.Dep('rmq.2', 'run', 'success', 'rmq_cluster.2', 'join')], + 'rmq.3': + [evapi.Dep('rmq.3', 'run', 'success', 'rmq_cluster.3', 'join')], 'rmq_cluster.1': [ - evapi.Dep('rmq_cluster.1', 'create', - 'success', 'rmq_cluster.2', 'join'), - evapi.Dep('rmq_cluster.1', 'create', 'success', 'rmq_cluster.3', 'join')]} + evapi.Dep('rmq_cluster.1', 'create', 'success', 'rmq_cluster.2', + 'join'), evapi.Dep('rmq_cluster.1', 'create', 'success', + 'rmq_cluster.3', 'join') + ] + } def test_rmq(rmq_deps): @@ -115,29 +117,35 @@ def test_rmq(rmq_deps): evapi.build_edges(changes_graph, rmq_deps) assert set(changes_graph.successors('rmq_cluster.1.create')) == { - 'rmq_cluster.2.join', 'rmq_cluster.3.join'} + 'rmq_cluster.2.join', 'rmq_cluster.3.join' + } def test_riak(): events = { 'riak_service1': [ - evapi.React('riak_service1', 'run', 'success', - 'riak_service2', 'run'), - evapi.React('riak_service1', 'run', 'success', 'riak_service3', 'run')], + evapi.React('riak_service1', 'run', 'success', 'riak_service2', + 'run'), evapi.React('riak_service1', 'run', 'success', + 'riak_service3', 'run') + ], 'riak_service3': [ - evapi.React('riak_service3', 'join', 'success', - 'riak_service1', 'commit'), - evapi.React('riak_service3', 'run', 'success', 'riak_service3', 'join')], + evapi.React('riak_service3', 'join', 'success', 'riak_service1', + 'commit'), + evapi.React('riak_service3', 'run', 'success', 'riak_service3', + 'join') + ], 'riak_service2': [ - evapi.React('riak_service2', 'run', 'success', - 'riak_service2', 'join'), - evapi.React('riak_service2', 'join', 'success', 'riak_service1', 'commit')], - + evapi.React('riak_service2', 'run', 'success', 'riak_service2', + 'join'), + evapi.React('riak_service2', 'join', 'success', 'riak_service1', + 'commit') + ], } changes_graph = nx.MultiDiGraph() changes_graph.add_node('riak_service1.run') evapi.build_edges(changes_graph, events) assert set(changes_graph.predecessors('riak_service1.commit')) == { - 'riak_service2.join', 'riak_service3.join'} + 'riak_service2.join', 'riak_service3.join' + } diff --git a/solar/test/test_graph_api.py b/solar/test/test_graph_api.py index 94f10a17..5596fc69 100644 --- a/solar/test/test_graph_api.py +++ b/solar/test/test_graph_api.py @@ -18,14 +18,12 @@ from pytest import fixture from solar.orchestration import graph from solar.orchestration.traversal import states -from solar.dblayer.model import ModelMeta @fixture def simple(): simple_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'orch_fixtures', + os.path.dirname(os.path.realpath(__file__)), 'orch_fixtures', 'simple.yaml') return graph.create_plan(simple_path) @@ -59,7 +57,13 @@ def test_wait_finish(simple): simple.node[n]['status'] = states.SUCCESS.name graph.update_graph(simple) assert next(graph.wait_finish(simple.graph['uid'], 10)) == { - 'SKIPPED': 0, 'SUCCESS': 2, 'NOOP': 0, 'ERROR': 0, 'INPROGRESS': 0, 'PENDING': 0} + 'SKIPPED': 0, + 'SUCCESS': 2, + 'NOOP': 0, + 'ERROR': 0, + 'INPROGRESS': 0, + 'PENDING': 0 + } def test_several_updates(simple): @@ -67,10 +71,22 @@ def test_several_updates(simple): graph.update_graph(simple) assert next(graph.wait_finish(simple.graph['uid'], 10)) == { - 'SKIPPED': 0, 'SUCCESS': 0, 'NOOP': 0, 'ERROR': 1, 'INPROGRESS': 0, 'PENDING': 1} + 'SKIPPED': 0, + 'SUCCESS': 0, + 'NOOP': 0, + 'ERROR': 1, + 'INPROGRESS': 0, + 'PENDING': 1 + } simple.node['echo_stuff']['status'] = states.ERROR.name graph.update_graph(simple) assert next(graph.wait_finish(simple.graph['uid'], 10)) == { - 'SKIPPED': 0, 'SUCCESS': 0, 'NOOP': 0, 'ERROR': 2, 'INPROGRESS': 0, 'PENDING': 0} + 'SKIPPED': 0, + 'SUCCESS': 0, + 'NOOP': 0, + 'ERROR': 2, + 'INPROGRESS': 0, + 'PENDING': 0 + } diff --git a/solar/test/test_graph_filtering.py b/solar/test/test_graph_filtering.py index df24ca51..06c9342a 100644 --- a/solar/test/test_graph_filtering.py +++ b/solar/test/test_graph_filtering.py @@ -14,14 +14,13 @@ import os +import networkx as nx from pytest import fixture from pytest import mark -import networkx as nx -from solar.orchestration import graph from solar.orchestration import filters +from solar.orchestration import graph from solar.orchestration.traversal import states -from solar.utils import yaml_load @fixture @@ -44,8 +43,7 @@ def test_end_at(dg_ex1, end_nodes, visited): @mark.parametrize("start_nodes,visited", [ - (['n3'], {'n3'}), - (['n1'], {'n1', 'n2', 'n4'}), + (['n3'], {'n3'}), (['n1'], {'n1', 'n2', 'n4'}), (['n1', 'n3'], {'n1', 'n2', 'n3', 'n4', 'n5'}) ]) def test_start_from(dg_ex1, start_nodes, visited): @@ -63,20 +61,21 @@ def dg_ex2(): @fixture def riak_plan(): riak_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'orch_fixtures', + os.path.dirname(os.path.realpath(__file__)), 'orch_fixtures', 'riak.yaml') return graph.create_plan(riak_path, save=False) def test_riak_start_node1(riak_plan): assert filters.start_from(riak_plan, ['node1.run']) == { - 'node1.run', 'hosts_file1.run', 'riak_service1.run'} + 'node1.run', 'hosts_file1.run', 'riak_service1.run' + } def test_riak_end_hosts_file1(riak_plan): assert filters.end_at(riak_plan, ['hosts_file1.run']) == { - 'node1.run', 'hosts_file1.run'} + 'node1.run', 'hosts_file1.run' + } def test_start_at_two_nodes(riak_plan): @@ -87,8 +86,10 @@ def test_start_at_two_nodes(riak_plan): def test_initial_from_node1_traverse(riak_plan): filters.filter(riak_plan, start=['node1.run']) - pending = {n for n in riak_plan if riak_plan.node[ - n]['status'] == states.PENDING.name} + pending = {n + for n in riak_plan + if riak_plan.node[ + n]['status'] == states.PENDING.name} assert pending == {'hosts_file1.run', 'riak_service1.run', 'node1.run'} @@ -97,18 +98,21 @@ def test_second_from_node2_with_node1_walked(riak_plan): for n in success: riak_plan.node[n]['status'] = states.SUCCESS.name filters.filter(riak_plan, start=['node2.run']) - pending = {n for n in riak_plan if riak_plan.node[ - n]['status'] == states.PENDING.name} - assert pending == {'hosts_file2.run', 'riak_service2.run', - 'node2.run', 'riak_service2.join'} + pending = {n + for n in riak_plan + if riak_plan.node[ + n]['status'] == states.PENDING.name} + assert pending == {'hosts_file2.run', 'riak_service2.run', 'node2.run', + 'riak_service2.join'} def test_end_joins(riak_plan): - filters.filter( - riak_plan, - start=['node1.run', 'node2.run', 'node3.run'], - end=['riak_service2.join', 'riak_service3.join']) - skipped = {n for n in riak_plan if riak_plan.node[ - n]['status'] == states.SKIPPED.name} + filters.filter(riak_plan, + start=['node1.run', 'node2.run', 'node3.run'], + end=['riak_service2.join', 'riak_service3.join']) + skipped = {n + for n in riak_plan + if riak_plan.node[ + n]['status'] == states.SKIPPED.name} assert skipped == {'riak_service1.commit'} diff --git a/solar/test/test_limits.py b/solar/test/test_limits.py index 130220fc..99a2e167 100644 --- a/solar/test/test_limits.py +++ b/solar/test/test_limits.py @@ -14,40 +14,49 @@ import os -from pytest import fixture import networkx as nx +from pytest import fixture -from solar.orchestration import limits from solar.orchestration import graph +from solar.orchestration import limits @fixture def dg(): ex = nx.DiGraph() - ex.add_node('t1', status='PENDING', target='1', - resource_type='node', type_limit=2) - ex.add_node('t2', status='PENDING', target='1', - resource_type='node', type_limit=2) - ex.add_node('t3', status='PENDING', target='1', - resource_type='node', type_limit=2) + ex.add_node('t1', + status='PENDING', + target='1', + resource_type='node', + type_limit=2) + ex.add_node('t2', + status='PENDING', + target='1', + resource_type='node', + type_limit=2) + ex.add_node('t3', + status='PENDING', + target='1', + resource_type='node', + type_limit=2) return ex def test_target_rule(dg): - assert limits.target_based_rule(dg, [], 't1') == True - assert limits.target_based_rule(dg, ['t1'], 't2') == False + assert limits.target_based_rule(dg, [], 't1') is True + assert limits.target_based_rule(dg, ['t1'], 't2') is False def test_type_limit_rule(dg): - assert limits.type_based_rule(dg, ['t1'], 't2') == True - assert limits.type_based_rule(dg, ['t1', 't2'], 't3') == False + assert limits.type_based_rule(dg, ['t1'], 't2') is True + assert limits.type_based_rule(dg, ['t1', 't2'], 't3') is False def test_items_rule(dg): assert limits.items_rule(dg, ['1'] * 99, '2') - assert limits.items_rule(dg, ['1'] * 99, '2', limit=10) == False + assert limits.items_rule(dg, ['1'] * 99, '2', limit=10) is False @fixture @@ -68,8 +77,7 @@ def test_filtering_chain(target_dg): @fixture def seq_plan(): seq_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), - 'orch_fixtures', + os.path.dirname(os.path.realpath(__file__)), 'orch_fixtures', 'sequential.yaml') return graph.create_plan(seq_path, save=False) @@ -78,6 +86,6 @@ def test_limits_sequential(seq_plan): stack_to_execute = seq_plan.nodes() while stack_to_execute: left = stack_to_execute[0] - assert list(limits.get_default_chain( - seq_plan, [], stack_to_execute)) == [left] + assert list(limits.get_default_chain(seq_plan, [], + stack_to_execute)) == [left] stack_to_execute.pop(0) diff --git a/solar/test/test_operations_with_tags.py b/solar/test/test_operations_with_tags.py index 67a25e39..af4ede47 100644 --- a/solar/test/test_operations_with_tags.py +++ b/solar/test/test_operations_with_tags.py @@ -15,8 +15,8 @@ from pytest import fixture from solar.core import resource -from solar.dblayer.solar_models import Resource from solar.dblayer.model import ModelMeta +from solar.dblayer.solar_models import Resource @fixture diff --git a/solar/test/test_resource.py b/solar/test/test_resource.py index fb1b81be..23ff54b1 100644 --- a/solar/test/test_resource.py +++ b/solar/test/test_resource.py @@ -13,13 +13,11 @@ # under the License. import base - from solar.core import resource from solar.core import signals class TestResource(base.BaseResourceTest): - def test_resource_args(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -31,9 +29,8 @@ input: value: 0 """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'value': 1} - ) + sample1 = self.create_resource('sample1', sample_meta_dir, + {'value': 1}) self.assertEqual(sample1.args['value'], 1) # test default value @@ -41,7 +38,8 @@ input: self.assertEqual(sample2.args['value'], 0) def test_connections_recreated_after_load(self): - """ + """Test if connections are ok after load + Create resource in some process. Then in other process load it. All connections should remain the same. """ @@ -56,12 +54,9 @@ input: """) def creating_process(): - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'value': 1} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, - ) + sample1 = self.create_resource('sample1', sample_meta_dir, + {'value': 1}) + sample2 = self.create_resource('sample2', sample_meta_dir, ) signals.connect(sample1, sample2) self.assertEqual(sample1.args['value'], sample2.args['value']) @@ -86,9 +81,7 @@ input: value: 0 """) - sample = self.create_resource( - 'sample', sample_meta_dir, {'value': 1} - ) + sample = self.create_resource('sample', sample_meta_dir, {'value': 1}) sample_l = resource.load('sample') @@ -107,12 +100,9 @@ input: value: 0 """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'value': 1} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {} - ) + sample1 = self.create_resource('sample1', sample_meta_dir, + {'value': 1}) + sample2 = self.create_resource('sample2', sample_meta_dir, {}) signals.connect(sample1, sample2) self.assertEqual(sample1.args['value'], sample2.args['value']) diff --git a/solar/test/test_signals.py b/solar/test/test_signals.py index bd0ac888..30407ecb 100644 --- a/solar/test/test_signals.py +++ b/solar/test/test_signals.py @@ -12,15 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. -import base - -from solar.core import signals as xs - import pytest +import base +from solar.core import signals as xs + class TestBaseInput(base.BaseResourceTest): - def test_no_self_connection(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -32,13 +30,11 @@ input: value: """) - sample = self.create_resource( - 'sample', sample_meta_dir, {'value': 'x'} - ) + sample = self.create_resource('sample', sample_meta_dir, + {'value': 'x'}) - with self.assertRaisesRegexp( - Exception, - 'Trying to connect value-.* to itself'): + with self.assertRaisesRegexp(Exception, + 'Trying to connect value-.* to itself'): xs.connect(sample, sample, {'value'}) def test_input_dict_type(self): @@ -52,42 +48,23 @@ input: value: {} """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'values': {'a': 1, 'b': 2}} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir - ) + sample1 = self.create_resource('sample1', sample_meta_dir, {'values': + {'a': 1, + 'b': 2}}) + sample2 = self.create_resource('sample2', sample_meta_dir) xs.connect(sample1, sample2) - self.assertEqual( - sample1.args['values'], - sample2.args['values'] - ) + self.assertEqual(sample1.args['values'], sample2.args['values']) # Check update sample1.update({'values': {'a': 2}}) - self.assertEqual( - sample1.args['values'], - {'a': 2} - ) - self.assertEqual( - sample1.args['values'], - sample2.args['values'], - ) + self.assertEqual(sample1.args['values'], {'a': 2}) + self.assertEqual(sample1.args['values'], sample2.args['values'], ) # Check disconnect # TODO: should sample2.value be reverted to original value? sample1.disconnect(sample2) sample1.update({'values': {'a': 3}}) - self.assertEqual( - sample1.args['values'], - {'a': 3} - ) - # self.assertEqual( - # sample2.args['values'], - # {'a': 2} - #) - #self.assertEqual(sample2.args['values'].emitter, None) + self.assertEqual(sample1.args['values'], {'a': 3}) def test_multiple_resource_disjoint_connect(self): sample_meta_dir = self.make_resource_meta(""" @@ -121,31 +98,19 @@ input: value: """) - sample = self.create_resource( - 'sample', sample_meta_dir, {'ip': None, 'port': None} - ) - sample_ip = self.create_resource( - 'sample-ip', sample_ip_meta_dir, {'ip': '10.0.0.1'} - ) - sample_port = self.create_resource( - 'sample-port', sample_port_meta_dir, {'port': 8000} - ) - self.assertNotEqual( - sample.resource_inputs()['ip'], - sample_ip.resource_inputs()['ip'], - ) + sample = self.create_resource('sample', sample_meta_dir, + {'ip': None, + 'port': None}) + sample_ip = self.create_resource('sample-ip', sample_ip_meta_dir, + {'ip': '10.0.0.1'}) + sample_port = self.create_resource('sample-port', sample_port_meta_dir, + {'port': 8000}) + self.assertNotEqual(sample.resource_inputs()['ip'], + sample_ip.resource_inputs()['ip'], ) xs.connect(sample_ip, sample) xs.connect(sample_port, sample) self.assertEqual(sample.args['ip'], sample_ip.args['ip']) self.assertEqual(sample.args['port'], sample_port.args['port']) - # self.assertEqual( - # sample.args['ip'].emitter, - # sample_ip.args['ip'] - #) - # self.assertEqual( - # sample.args['port'].emitter, - # sample_port.args['port'] - #) def test_simple_observer_unsubscription(self): sample_meta_dir = self.make_resource_meta(""" @@ -158,29 +123,18 @@ input: value: """) - sample = self.create_resource( - 'sample', sample_meta_dir, {'ip': None} - ) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'ip': '10.0.0.1'} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'ip': '10.0.0.2'} - ) + sample = self.create_resource('sample', sample_meta_dir, {'ip': None}) + sample1 = self.create_resource('sample1', sample_meta_dir, + {'ip': '10.0.0.1'}) + sample2 = self.create_resource('sample2', sample_meta_dir, + {'ip': '10.0.0.2'}) xs.connect(sample1, sample) self.assertEqual(sample1.args['ip'], sample.args['ip']) - #self.assertEqual(len(list(sample1.args['ip'].receivers)), 1) - # self.assertEqual( - # sample.args['ip'].emitter, - # sample1.args['ip'] - #) xs.connect(sample2, sample) self.assertEqual(sample2.args['ip'], sample.args['ip']) # sample should be unsubscribed from sample1 and subscribed to sample2 - #self.assertEqual(len(list(sample1.args['ip'].receivers)), 0) - #self.assertEqual(sample.args['ip'].emitter, sample2.args['ip']) sample2.update({'ip': '10.0.0.3'}) self.assertEqual(sample2.args['ip'], sample.args['ip']) @@ -198,20 +152,17 @@ input: value: """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'ip': '10.0.0.1'} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'ip': '10.0.0.2'} - ) + sample1 = self.create_resource('sample1', sample_meta_dir, + {'ip': '10.0.0.1'}) + sample2 = self.create_resource('sample2', sample_meta_dir, + {'ip': '10.0.0.2'}) xs.connect(sample1, sample2) - with self.assertRaises(Exception): + with self.assertRaises(Exception): # NOQA xs.connect(sample2, sample1) class TestListInput(base.BaseResourceTest): - def test_list_input_single(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -232,68 +183,36 @@ input: value: [] """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'ip': '10.0.0.1'} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'ip': '10.0.0.2'} - ) + sample1 = self.create_resource('sample1', sample_meta_dir, + {'ip': '10.0.0.1'}) + sample2 = self.create_resource('sample2', sample_meta_dir, + {'ip': '10.0.0.2'}) list_input_single = self.create_resource( - 'list-input-single', list_input_single_meta_dir, {'ips': []} - ) + 'list-input-single', list_input_single_meta_dir, {'ips': []}) sample1.connect(list_input_single, mapping={'ip': 'ips'}) - self.assertItemsEqual( - #[ip['value'] for ip in list_input_single.args['ips']], - list_input_single.args['ips'], - [ - sample1.args['ip'], - ] - ) - # self.assertListEqual( - # [(e['emitter_attached_to'], e['emitter']) for e in list_input_single.args['ips']], - # [(sample1.args['ip'].attached_to.name, 'ip')] - #) + self.assertItemsEqual(list_input_single.args['ips'], [ + sample1.args['ip'], + ]) sample2.connect(list_input_single, mapping={'ip': 'ips'}) - self.assertItemsEqual( - #[ip['value'] for ip in list_input_single.args['ips']], - list_input_single.args['ips'], - [ - sample1.args['ip'], - sample2.args['ip'], - ] - ) - # self.assertListEqual( - # [(e['emitter_attached_to'], e['emitter']) for e in list_input_single.args['ips']], - # [(sample1.args['ip'].attached_to.name, 'ip'), - # (sample2.args['ip'].attached_to.name, 'ip')] - #) + self.assertItemsEqual(list_input_single.args['ips'], [ + sample1.args['ip'], + sample2.args['ip'], + ]) # Test update sample2.update({'ip': '10.0.0.3'}) - self.assertItemsEqual( - #[ip['value'] for ip in list_input_single.args['ips']], - list_input_single.args['ips'], - [ - sample1.args['ip'], - sample2.args['ip'], - ] - ) + self.assertItemsEqual(list_input_single.args['ips'], [ + sample1.args['ip'], + sample2.args['ip'], + ]) # Test disconnect sample2.disconnect(list_input_single) - self.assertItemsEqual( - #[ip['value'] for ip in list_input_single.args['ips']], - list_input_single.args['ips'], - [ - sample1.args['ip'], - ] - ) - # self.assertListEqual( - # [(e['emitter_attached_to'], e['emitter']) for e in list_input_single.args['ips']], - # [(sample1.args['ip'].attached_to.name, 'ip')] - #) + self.assertItemsEqual(list_input_single.args['ips'], [ + sample1.args['ip'], + ]) def test_list_input_multi(self): sample_meta_dir = self.make_resource_meta(""" @@ -321,70 +240,58 @@ input: value: """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, {'ip': '10.0.0.1', 'port': 1000} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, {'ip': '10.0.0.2', 'port': 1001} - ) - list_input_multi = self.create_resource( - 'list-input-multi', list_input_multi_meta_dir, args={'ips': [], 'ports': []} - ) + sample1 = self.create_resource('sample1', sample_meta_dir, + {'ip': '10.0.0.1', + 'port': 1000}) + sample2 = self.create_resource('sample2', sample_meta_dir, + {'ip': '10.0.0.2', + 'port': 1001}) + list_input_multi = self.create_resource('list-input-multi', + list_input_multi_meta_dir, + args={'ips': [], + 'ports': []}) - xs.connect(sample1, list_input_multi, mapping={ - 'ip': 'ips', 'port': 'ports'}) + xs.connect(sample1, + list_input_multi, + mapping={ + 'ip': 'ips', + 'port': 'ports' + }) self.assertItemsEqual( - #[ip['value'] for ip in list_input_multi.args['ips']], list_input_multi.args['ips'], - [sample1.args['ip']] - ) + [sample1.args['ip']]) self.assertItemsEqual( - #[p['value'] for p in list_input_multi.args['ports']], list_input_multi.args['ports'], - [sample1.args['port']] - ) + [sample1.args['port']]) - xs.connect(sample2, list_input_multi, mapping={ - 'ip': 'ips', 'port': 'ports'}) + xs.connect(sample2, + list_input_multi, + mapping={ + 'ip': 'ips', + 'port': 'ports' + }) self.assertItemsEqual( - #[ip['value'] for ip in list_input_multi.args['ips']], list_input_multi.args['ips'], [ sample1.args['ip'], sample2.args['ip'], - ] - ) - # self.assertListEqual( - # [(e['emitter_attached_to'], e['emitter']) for e in list_input_multi.args['ips']], - # [(sample1.args['ip'].attached_to.name, 'ip'), - # (sample2.args['ip'].attached_to.name, 'ip')] - #) + ]) + self.assertItemsEqual( - #[p['value'] for p in list_input_multi.args['ports']], list_input_multi.args['ports'], [ sample1.args['port'], sample2.args['port'], - ] - ) - # self.assertListEqual( - # [(e['emitter_attached_to'], e['emitter']) for e in list_input_multi.args['ports']], - # [(sample1.args['port'].attached_to.name, 'port'), - # (sample2.args['port'].attached_to.name, 'port')] - #) + ]) # Test disconnect sample2.disconnect(list_input_multi) self.assertItemsEqual( - #[ip['value'] for ip in list_input_multi.args['ips']], list_input_multi.args['ips'], - [sample1.args['ip']] - ) + [sample1.args['ip']]) self.assertItemsEqual( - #[p['value'] for p in list_input_multi.args['ports']], list_input_multi.args['ports'], - [sample1.args['port']] - ) + [sample1.args['port']]) # XXX: not used for now, not implemented in new db (jnowak) # @pytest.mark.xfail(reason="Nested lists are not supported in new_db") @@ -446,14 +353,13 @@ input: # sample1.connect(list_input, mapping={'ip': 'ips', 'port': 'ports'}) # sample2.connect(list_input, mapping={'ip': 'ips', 'port': 'ports'}) -# list_input.connect(list_input_nested, mapping={'ips': 'ipss', 'ports': 'portss'}) +# list_input.connect(list_input_nested, +# mapping={'ips': 'ipss', 'ports': 'portss'}) # self.assertListEqual( -# #[ips['value'] for ips in list_input_nested.args['ipss']], # list_input_nested.args['ipss'], # [list_input.args['ips']] # ) # self.assertListEqual( -# #[ps['value'] for ps in list_input_nested.args['portss']], # list_input_nested.args['portss'], # [list_input.args['ports']] # ) @@ -461,7 +367,6 @@ input: # # Test disconnect # xs.disconnect(sample1, list_input) # self.assertListEqual( -# #[[ip['value'] for ip in ips['value']] for ips in list_input_nested.args['ipss']], # list_input_nested.args['ipss'], # [[sample2.args['ip']]] # ) @@ -472,8 +377,8 @@ input: class TestHashInput(base.BaseResourceTest): - - @pytest.mark.xfail(reason="Connect should raise an error if already connected") + @pytest.mark.xfail( + reason="Connect should raise an error if already connected") def test_hash_input_basic(self): sample_meta_dir = self.make_resource_meta(""" id: sample @@ -496,42 +401,51 @@ input: schema: {ip: str!, port: int!} """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, args={'ip': '10.0.0.1', 'port': 5000} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, args={'ip': '10.0.0.2', 'port': 5001} - ) - sample3 = self.create_resource( - 'sample3', sample_meta_dir, args={'ip': '10.0.0.3', 'port': 5002} - ) - receiver = self.create_resource( - 'receiver', receiver_meta_dir - ) - xs.connect(sample1, receiver, mapping={ - 'ip': 'server:ip', 'port': 'server:port'}) + sample1 = self.create_resource('sample1', + sample_meta_dir, + args={'ip': '10.0.0.1', + 'port': 5000}) + sample2 = self.create_resource('sample2', + sample_meta_dir, + args={'ip': '10.0.0.2', + 'port': 5001}) + sample3 = self.create_resource('sample3', + sample_meta_dir, + args={'ip': '10.0.0.3', + 'port': 5002}) + receiver = self.create_resource('receiver', receiver_meta_dir) + xs.connect(sample1, + receiver, + mapping={ + 'ip': 'server:ip', + 'port': 'server:port' + }) self.assertDictEqual( - {'ip': sample1.args['ip'], 'port': sample1.args['port']}, - receiver.args['server'], - ) + {'ip': sample1.args['ip'], + 'port': sample1.args['port']}, + receiver.args['server'], ) sample1.update({'ip': '10.0.0.2'}) self.assertDictEqual( - {'ip': sample1.args['ip'], 'port': sample1.args['port']}, - receiver.args['server'], - ) + {'ip': sample1.args['ip'], + 'port': sample1.args['port']}, + receiver.args['server'], ) # XXX: We need to disconnect first # XXX: it should raise error when connecting already connected inputs xs.connect(sample2, receiver, mapping={'ip': 'server:ip'}) self.assertDictEqual( - {'ip': sample2.args['ip'], 'port': sample1.args['port']}, - receiver.args['server'], - ) - xs.connect(sample3, receiver, mapping={ - 'ip': 'server:ip', 'port': 'server:port'}) + {'ip': sample2.args['ip'], + 'port': sample1.args['port']}, + receiver.args['server'], ) + xs.connect(sample3, + receiver, + mapping={ + 'ip': 'server:ip', + 'port': 'server:port' + }) self.assertDictEqual( - {'ip': sample3.args['ip'], 'port': sample3.args['port']}, - receiver.args['server'], - ) + {'ip': sample3.args['ip'], + 'port': sample3.args['port']}, + receiver.args['server'], ) def test_hash_input_mixed(self): sample_meta_dir = self.make_resource_meta(""" @@ -555,22 +469,23 @@ input: schema: {ip: str!, port: int!} """) - sample = self.create_resource( - 'sample', sample_meta_dir, args={'ip': '10.0.0.1', 'port': 5000} - ) - receiver = self.create_resource( - 'receiver', receiver_meta_dir, args={'server': {'port': 5001}} - ) + sample = self.create_resource('sample', + sample_meta_dir, + args={'ip': '10.0.0.1', + 'port': 5000}) + receiver = self.create_resource('receiver', + receiver_meta_dir, + args={'server': {'port': 5001}}) sample.connect(receiver, mapping={'ip': 'server:ip'}) self.assertDictEqual( - {'ip': sample.args['ip'], 'port': 5001}, - receiver.args['server'], - ) + {'ip': sample.args['ip'], + 'port': 5001}, + receiver.args['server'], ) sample.update({'ip': '10.0.0.2'}) self.assertDictEqual( - {'ip': sample.args['ip'], 'port': 5001}, - receiver.args['server'], - ) + {'ip': sample.args['ip'], + 'port': 5001}, + receiver.args['server'], ) def test_hash_input_with_list(self): sample_meta_dir = self.make_resource_meta(""" @@ -594,33 +509,41 @@ input: schema: [{ip: str!, port: int!}] """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, args={'ip': '10.0.0.1', 'port': 5000} - ) - receiver = self.create_resource( - 'receiver', receiver_meta_dir - ) - xs.connect(sample1, receiver, mapping={ - 'ip': 'server:ip', 'port': 'server:port'}) + sample1 = self.create_resource('sample1', + sample_meta_dir, + args={'ip': '10.0.0.1', + 'port': 5000}) + receiver = self.create_resource('receiver', receiver_meta_dir) + xs.connect(sample1, + receiver, + mapping={ + 'ip': 'server:ip', + 'port': 'server:port' + }) self.assertItemsEqual( - [{'ip': sample1.args['ip'], 'port': sample1.args['port']}], - receiver.args['server'], - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, args={'ip': '10.0.0.2', 'port': 5001} - ) - xs.connect(sample2, receiver, mapping={ - 'ip': 'server:ip', 'port': 'server:port'}) + [{'ip': sample1.args['ip'], + 'port': sample1.args['port']}], + receiver.args['server'], ) + sample2 = self.create_resource('sample2', + sample_meta_dir, + args={'ip': '10.0.0.2', + 'port': 5001}) + xs.connect(sample2, + receiver, + mapping={ + 'ip': 'server:ip', + 'port': 'server:port' + }) self.assertItemsEqual( - [{'ip': sample1.args['ip'], 'port': sample1.args['port']}, - {'ip': sample2.args['ip'], 'port': sample2.args['port']}], - receiver.args['server'], - ) + [{'ip': sample1.args['ip'], + 'port': sample1.args['port']}, {'ip': sample2.args['ip'], + 'port': sample2.args['port']}], + receiver.args['server'], ) sample1.disconnect(receiver) self.assertItemsEqual( - [{'ip': sample2.args['ip'], 'port': sample2.args['port']}], - receiver.args['server'], - ) + [{'ip': sample2.args['ip'], + 'port': sample2.args['port']}], + receiver.args['server'], ) def test_hash_input_with_multiple_connections(self): sample_meta_dir = self.make_resource_meta(""" @@ -644,21 +567,14 @@ input: schema: {ip: str!} """) - sample = self.create_resource( - 'sample', sample_meta_dir, args={'ip': '10.0.0.1'} - ) - receiver = self.create_resource( - 'receiver', receiver_meta_dir - ) + sample = self.create_resource('sample', + sample_meta_dir, + args={'ip': '10.0.0.1'}) + receiver = self.create_resource('receiver', receiver_meta_dir) xs.connect(sample, receiver, mapping={'ip': ['ip', 'server:ip']}) - self.assertEqual( - sample.args['ip'], - receiver.args['ip'] - ) - self.assertDictEqual( - {'ip': sample.args['ip']}, - receiver.args['server'], - ) + self.assertEqual(sample.args['ip'], receiver.args['ip']) + self.assertDictEqual({'ip': sample.args['ip']}, + receiver.args['server'], ) def test_hash_input_multiple_resources_with_tag_connect(self): sample_meta_dir = self.make_resource_meta(""" @@ -682,44 +598,49 @@ input: schema: [{ip: str!, port: int!}] """) - sample1 = self.create_resource( - 'sample1', sample_meta_dir, args={'ip': '10.0.0.1', 'port': 5000} - ) - sample2 = self.create_resource( - 'sample2', sample_meta_dir, args={'ip': '10.0.0.2', 'port': 5001} - ) - receiver = self.create_resource( - 'receiver', receiver_meta_dir - ) + sample1 = self.create_resource('sample1', + sample_meta_dir, + args={'ip': '10.0.0.1', + 'port': 5000}) + sample2 = self.create_resource('sample2', + sample_meta_dir, + args={'ip': '10.0.0.2', + 'port': 5001}) + receiver = self.create_resource('receiver', receiver_meta_dir) sample1.connect(receiver, mapping={'ip': 'server:ip'}) sample2.connect(receiver, mapping={'port': 'server:port|sample1'}) self.assertItemsEqual( - [{'ip': sample1.args['ip'], 'port': sample2.args['port']}], - receiver.args['server'], - ) - sample3 = self.create_resource( - 'sample3', sample_meta_dir, args={'ip': '10.0.0.3', 'port': 5002} - ) - sample3.connect(receiver, mapping={ - 'ip': 'server:ip', 'port': 'server:port'}) + [{'ip': sample1.args['ip'], + 'port': sample2.args['port']}], + receiver.args['server'], ) + sample3 = self.create_resource('sample3', + sample_meta_dir, + args={'ip': '10.0.0.3', + 'port': 5002}) + sample3.connect(receiver, + mapping={ + 'ip': 'server:ip', + 'port': 'server:port' + }) self.assertItemsEqual( - [{'ip': sample1.args['ip'], 'port': sample2.args['port']}, - {'ip': sample3.args['ip'], 'port': sample3.args['port']}], - receiver.args['server'], - ) - sample4 = self.create_resource( - 'sample4', sample_meta_dir, args={'ip': '10.0.0.4', 'port': 5003} - ) + [{'ip': sample1.args['ip'], + 'port': sample2.args['port']}, {'ip': sample3.args['ip'], + 'port': sample3.args['port']}], + receiver.args['server'], ) + sample4 = self.create_resource('sample4', + sample_meta_dir, + args={'ip': '10.0.0.4', + 'port': 5003}) sample4.connect(receiver, mapping={'port': 'server:port|sample3'}) self.assertItemsEqual( - [{'ip': sample1.args['ip'], 'port': sample2.args['port']}, - {'ip': sample3.args['ip'], 'port': sample4.args['port']}], - receiver.args['server'], - ) + [{'ip': sample1.args['ip'], + 'port': sample2.args['port']}, {'ip': sample3.args['ip'], + 'port': sample4.args['port']}], + receiver.args['server'], ) # There can be no sample3 connections left now sample4.connect(receiver, mapping={'ip': 'server:ip|sample3'}) self.assertItemsEqual( - [{'ip': sample1.args['ip'], 'port': sample2.args['port']}, - {'ip': sample4.args['ip'], 'port': sample4.args['port']}], - receiver.args['server'], - ) + [{'ip': sample1.args['ip'], + 'port': sample2.args['port']}, {'ip': sample4.args['ip'], + 'port': sample4.args['port']}], + receiver.args['server'], ) diff --git a/solar/test/test_system_log_api.py b/solar/test/test_system_log_api.py index 5d897fbc..90b2ec60 100644 --- a/solar/test/test_system_log_api.py +++ b/solar/test/test_system_log_api.py @@ -13,25 +13,27 @@ # under the License. import mock -from pytest import fixture from pytest import mark +from solar.core.resource import resource +from solar.core.resource import RESOURCE_STATE +from solar.core import signals +from solar.dblayer.model import ModelMeta +from solar.dblayer.solar_models import CommitedResource +from solar.dblayer.solar_models import Resource as DBResource from solar.system_log import change from solar.system_log import data from solar.system_log import operations -from solar.core import signals -from solar.core.resource import resource, RESOURCE_STATE -from solar.dblayer.solar_models import Resource as DBResource -from solar.dblayer.solar_models import CommitedResource -from solar.dblayer.model import ModelMeta def test_revert_update(): commit = {'a': '10'} previous = {'a': '9'} res = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + {'name': 'test1', + 'base_path': 'x', + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res.save() action = 'update' res.inputs['a'] = '9' @@ -40,9 +42,11 @@ def test_revert_update(): assert resource_obj.args == previous log = data.SL() - logitem = change.create_logitem( - res.name, action, change.create_diff(commit, previous), [], - base_path=res.base_path) + logitem = change.create_logitem(res.name, + action, + change.create_diff(commit, previous), + [], + base_path=res.base_path) log.append(logitem) resource_obj.update(commit) operations.move_to_commited(logitem.log_action) @@ -56,23 +60,29 @@ def test_revert_update(): def test_revert_update_connected(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', + {'name': 'test1', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() res2 = DBResource.from_dict('test2', - {'name': 'test2', 'base_path': 'x', + {'name': 'test2', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res2.inputs['a'] = '' res2.save_lazy() res3 = DBResource.from_dict('test3', - {'name': 'test3', 'base_path': 'x', + {'name': 'test3', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res3.inputs['a'] = '' res3.save_lazy() @@ -113,15 +123,16 @@ def test_revert_update_connected(): def test_revert_removal(): res = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', + {'name': 'test1', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res.inputs['a'] = '9' res.save_lazy() - commited = CommitedResource.from_dict('test1', - {'inputs': {'a': '9'}, - 'state': 'operational'}) + commited = CommitedResource.from_dict('test1', {'inputs': {'a': '9'}, + 'state': 'operational'}) commited.save_lazy() resource_obj = resource.load(res.name) @@ -139,23 +150,31 @@ def test_revert_removal(): with mock.patch.object(resource, 'read_meta') as mread: mread.return_value = { - 'input': {'a': {'schema': 'str!'}}, 'id': 'mocked'} + 'input': {'a': {'schema': 'str!'}}, + 'id': 'mocked' + } change.revert(changes[0].uid) ModelMeta.save_all_lazy() assert len(DBResource.bucket.get('test1').siblings) == 1 resource_obj = resource.load('test1') assert resource_obj.args == { - 'a': '9', 'location_id': '', 'transports_id': ''} + 'a': '9', + 'location_id': '', + 'transports_id': '' + } -@mark.xfail(reason='With current approach child will be notice changes after parent is removed') +@mark.xfail( + reason="""With current approach child will + notice changes after parent is removed""" +) def test_revert_removed_child(): - res1 = orm.DBResource(id='test1', name='test1', base_path='x') + res1 = orm.DBResource(id='test1', name='test1', base_path='x') # NOQA res1.save() res1.add_input('a', 'str', '9') - res2 = orm.DBResource(id='test2', name='test2', base_path='x') + res2 = orm.DBResource(id='test2', name='test2', base_path='x') # NOQA res2.save() res2.add_input('a', 'str', 0) @@ -184,9 +203,11 @@ def test_revert_removed_child(): def test_revert_create(): res = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', + {'name': 'test1', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res.inputs['a'] = '9' res.save_lazy() ModelMeta.save_all_lazy() @@ -211,16 +232,20 @@ def test_revert_create(): def test_discard_all_pending_changes_resources_created(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', + {'name': 'test1', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() res2 = DBResource.from_dict('test2', - {'name': 'test2', 'base_path': 'x', + {'name': 'test2', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res2.inputs['a'] = '0' res2.save_lazy() ModelMeta.save_all_lazy() @@ -236,16 +261,20 @@ def test_discard_all_pending_changes_resources_created(): def test_discard_connection(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', + {'name': 'test1', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() res2 = DBResource.from_dict('test2', - {'name': 'test2', 'base_path': 'x', + {'name': 'test2', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res2.inputs['a'] = '0' res2.save_lazy() ModelMeta.save_all_lazy() @@ -267,9 +296,11 @@ def test_discard_connection(): def test_discard_removed(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', + {'name': 'test1', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() ModelMeta.save_all_lazy() @@ -289,9 +320,11 @@ def test_discard_removed(): def test_discard_update(): res1 = DBResource.from_dict('test1', - {'name': 'test1', 'base_path': 'x', + {'name': 'test1', + 'base_path': 'x', 'state': RESOURCE_STATE.created.name, - 'meta_inputs': {'a': {'value': None, 'schema': 'str'}}}) + 'meta_inputs': {'a': {'value': None, + 'schema': 'str'}}}) res1.inputs['a'] = '9' res1.save_lazy() ModelMeta.save_all_lazy() diff --git a/solar/test/test_validation.py b/solar/test/test_validation.py index acf67b9c..560a6044 100644 --- a/solar/test/test_validation.py +++ b/solar/test/test_validation.py @@ -12,10 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -from solar.test import base - -from solar import errors from solar.core import validation as sv +from solar.test import base class TestInputValidation(base.BaseResourceTest): diff --git a/solar/test/test_virtual_resource.py b/solar/test/test_virtual_resource.py index 8e807517..30d98d7c 100644 --- a/solar/test/test_virtual_resource.py +++ b/solar/test/test_virtual_resource.py @@ -18,8 +18,9 @@ from StringIO import StringIO import pytest import yaml -from solar.events.controls import React, Dep from solar.core.resource import virtual_resource as vr +from solar.events.controls import Dep +from solar.events.controls import React @pytest.fixture