From 3a520dd5443bc1e9928392a745347d4d5069f059 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Mon, 6 Jul 2015 15:14:44 +0200 Subject: [PATCH 1/9] Operations: fix updating of resource data with stage/commit --- run_tests.sh | 2 +- solar/solar/core/actions.py | 2 +- solar/solar/core/resource.py | 11 +++ solar/solar/operations.py | 2 + solar/solar/test/base.py | 4 +- .../solar/test/test_stage_commit_procedure.py | 3 +- .../solar/test/test_update_propagated_data.py | 75 ++++++++++++++++--- 7 files changed, 84 insertions(+), 15 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index fa31e702..f2709cba 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -20,6 +20,6 @@ pip install -r solar/requirements.txt --download-cache=/tmp/$JOB_NAME pushd solar/solar -PYTHONPATH=$WORKSPACE/solar CONFIG_FILE=$CONFIG_FILE py.test test/ +PYTHONPATH=$WORKSPACE/solar CONFIG_FILE=$CONFIG_FILE py.test -s test/ popd diff --git a/solar/solar/core/actions.py b/solar/solar/core/actions.py index eb209f11..98e8618c 100644 --- a/solar/solar/core/actions.py +++ b/solar/solar/core/actions.py @@ -3,7 +3,7 @@ import handlers def resource_action(resource, action): - handler = resource.metadata['handler'] + handler = resource.metadata.get('handler', 'none') with handlers.get(handler)([resource]) as h: return h.action(resource, action) diff --git a/solar/solar/core/resource.py b/solar/solar/core/resource.py index 245b1b9a..f37f1d8f 100644 --- a/solar/solar/core/resource.py +++ b/solar/solar/core/resource.py @@ -67,6 +67,8 @@ class Resource(object): 'Argument {} not implemented for resource {}'.format(k, self) ) + if isinstance(v, dict) and 'value' in v: + v = v['value'] self.metadata['input'][k]['value'] = v db.save(self.name, self.metadata, collection=db.COLLECTIONS.resource) @@ -168,6 +170,15 @@ def wrap_resource(raw_resource): return Resource(name, raw_resource, args, tags=tags, virtual_resource=virtual_resource) +def wrap_resource_no_value(raw_resource): + name = raw_resource['id'] + args = {k: v for k, v in raw_resource['input'].items()} + tags = raw_resource.get('tags', []) + virtual_resource = raw_resource.get('virtual_resource', []) + + return Resource(name, raw_resource, args, tags=tags, virtual_resource=virtual_resource) + + def load(resource_name): raw_resource = db.read(resource_name, collection=db.COLLECTIONS.resource) diff --git a/solar/solar/operations.py b/solar/solar/operations.py index 5a94e42e..e484164c 100644 --- a/solar/solar/operations.py +++ b/solar/solar/operations.py @@ -116,6 +116,8 @@ def commit(li, resources, commited, history): commited[li.res]['metadata']) result_state = execute(commited_res, 'remove') + staged_res.set_args_from_dict(staged_data['input']) + if result_state is state.STATES.success: result_state = execute(staged_res, 'run') else: diff --git a/solar/solar/test/base.py b/solar/solar/test/base.py index 5bb526ef..2a6d4c3d 100644 --- a/solar/solar/test/base.py +++ b/solar/solar/test/base.py @@ -4,7 +4,7 @@ import tempfile import unittest import yaml -from solar.core import resource as xr +from solar.core import virtual_resource as vr from solar.core import signals as xs from solar.interfaces.db import get_db @@ -31,4 +31,4 @@ class BaseResourceTest(unittest.TestCase): return path def create_resource(self, name, src, args): - return xr.create(name, src, args) + return vr.create(name, src, args)[0] diff --git a/solar/solar/test/test_stage_commit_procedure.py b/solar/solar/test/test_stage_commit_procedure.py index 6ac6aba8..3d74273c 100644 --- a/solar/solar/test/test_stage_commit_procedure.py +++ b/solar/solar/test/test_stage_commit_procedure.py @@ -15,7 +15,8 @@ def default_resources(): {'id': 'node1', 'input': {'ip': {'value':'10.0.0.3'}}}) rabbitmq_service1 = resource.wrap_resource( - {'id':'rabbitmq', 'input': { + {'id':'rabbitmq', + 'input': { 'ip' : {'value': ''}, 'image': {'value': 'rabbitmq:3-management'}}}) signals.connect(node1, rabbitmq_service1) diff --git a/solar/solar/test/test_update_propagated_data.py b/solar/solar/test/test_update_propagated_data.py index 007e4109..af3d21f4 100644 --- a/solar/solar/test/test_update_propagated_data.py +++ b/solar/solar/test/test_update_propagated_data.py @@ -11,7 +11,8 @@ def resources(): {'id': 'node1', 'input': {'ip': {'value': '10.0.0.3'}}}) mariadb_service1 = resource.wrap_resource( - {'id': 'mariadb', 'input': { + {'id': 'mariadb', + 'input': { 'port' : {'value': 3306}, 'ip': {'value': ''}}}) keystone_db = resource.wrap_resource( @@ -50,29 +51,85 @@ def test_update_port_on_mariadb(resources): ('change', u'metadata.input.login_port.value', (3306, 4400))] +@pytest.fixture +def simple_input(): + res1 = resource.wrap_resource( + {'id': 'res1', + 'input': {'ip': {'value': '10.10.0.2'}}}) + res2 = resource.wrap_resource( + {'id': 'res2', + 'input': {'ip': {'value': '10.10.0.3'}}}) + + signals.connect(res1, res2) + return resource.load_all() + + +def test_update_simple_resource(simple_input): + operations.stage_changes() + operations.commit_changes() + + res1 = simple_input['res1'] + res1.update({'ip': '10.0.0.3'}) + + log = operations.stage_changes() + + assert len(log) == 2 + + assert log.items[0].diff == [ + ('change', u'input.ip.value', ('10.10.0.2', '10.0.0.3')), + ('change', 'metadata.input.ip.value', ('10.10.0.2', '10.0.0.3')), + ] + assert log.items[1].diff == [ + ('change', u'input.ip.value', ('10.10.0.2', '10.0.0.3')), + ('change', 'metadata.input.ip.value', ('10.10.0.2', '10.0.0.3')), + ] + + operations.commit_changes() + assert simple_input['res1'].args_dict() == { + 'ip': '10.0.0.3', + } + assert simple_input['res2'].args_dict() == { + 'ip': '10.0.0.3', + } + + log_item = operations.rollback_last() + assert log_item.diff == [ + ('change', u'input.ip.value', (u'10.0.0.3', u'10.10.0.2')), + ('change', 'metadata.input.ip.value', ('10.0.0.3', '10.10.0.2')), + ] + + res2 = resource.load('res2') + assert res2.args_dict() == { + 'ip': '10.10.0.2', + } + + @pytest.fixture def list_input(): res1 = resource.wrap_resource( - {'id': 'res1', 'input': {'ip': {'value': '10.10.0.2'}}}) + {'id': 'res1', + 'input': {'ip': {'value': '10.10.0.2'}}}) res2 = resource.wrap_resource( - {'id': 'res2', 'input': {'ip': {'value': '10.10.0.3'}}}) + {'id': 'res2', + 'input': {'ip': {'value': '10.10.0.3'}}}) consumer = resource.wrap_resource( - {'id': 'consumer', 'input': - {'ips': {'value': [], - 'schema': ['str']}}}) + {'id': 'consumer', + 'input': + {'ips': {'value': [], + 'schema': ['str']}}}) signals.connect(res1, consumer, {'ip': 'ips'}) signals.connect(res2, consumer, {'ip': 'ips'}) return resource.load_all() -@pytest.mark.xfail def test_update_list_resource(list_input): operations.stage_changes() operations.commit_changes() res3 = resource.wrap_resource( - {'id': 'res3', 'input': {'ip': {'value': '10.10.0.4'}}}) + {'id': 'res3', + 'input': {'ip': {'value': '10.10.0.4'}}}) signals.connect(res3, list_input['consumer'], {'ip': 'ips'}) log = operations.stage_changes() @@ -110,5 +167,3 @@ def test_update_list_resource(list_input): {u'emitter': u'ip', u'emitter_attached_to': u'res2', u'value': u'10.10.0.3'}]} - - From f6cf5969b1da0a840ceb951ab82e9d32facc12ca Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 7 Jul 2015 08:49:35 +0200 Subject: [PATCH 2/9] cli.py: path validation, show resource by name option added --- solar/solar/cli.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/solar/solar/cli.py b/solar/solar/cli.py index b2a6faee..bf759e22 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -286,7 +286,7 @@ def init_cli_resource(): @resource.command() @click.argument('name') - @click.argument('base_path') + @click.argument('base_path', type=click.Path(exists=True, file_okay=False)) @click.argument('args') def create(args, base_path, name): click.echo('create {} {} {}'.format(name, base_path, args)) @@ -296,41 +296,48 @@ def init_cli_resource(): print res.name @resource.command() + @click.option('--name', default=None) @click.option('--tag', default=None) - @click.option('--use-json/--no-use-json', default=False) - @click.option('--color/--no-color', default=True) - def show(color, use_json, tag): + @click.option('--json', default=False, is_flag=True) + @click.option('--color', default=True, is_flag=True) + def show(**kwargs): resources = [] for name, res in sresource.load_all().items(): show = True - if tag: - if tag not in res.tags: + if kwargs['tag']: + print res.name, res.tags + if kwargs['tag'] not in res.tags: + show = False + if kwargs['name']: + if res.name != kwargs['name']: show = False if show: resources.append(res) - if use_json: + echo = click.echo_via_pager + if kwargs['json']: output = json.dumps([r.to_dict() for r in resources], indent=2) + echo = click.echo else: - if color: + if kwargs['color']: formatter = lambda r: r.color_repr() else: formatter = lambda r: unicode(r) output = '\n'.join(formatter(r) for r in resources) if output: - click.echo_via_pager(output) + echo(output) @resource.command() - @click.argument('resource_path') + @click.argument('resource_name') @click.argument('tag_name') @click.option('--add/--delete', default=True) - def tag(add, tag_name, resource_path): - click.echo('Tag {} with {} {}'.format(resource_path, tag_name, add)) - r = sresource.load(resource_path) + def tag(add, tag_name, resource_name): + click.echo('Tag {} with {} {}'.format(resource_name, tag_name, add)) + r = sresource.load(resource_name) if add: r.add_tag(tag_name) else: @@ -354,7 +361,7 @@ def init_cli_resource(): print 'ERROR: %s: %s' % (r.name, error) @resource.command() - @click.argument('path') + @click.argument('path', type=click.Path(exists=True, dir_okay=False)) def get_inputs(path): with open(path) as f: content = f.read() From 21b8332ef13a9c9b46b00dadfc3a29ca52839521 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 7 Jul 2015 08:52:44 +0200 Subject: [PATCH 3/9] README updated, fix for printing in resource.show --- README.md | 4 +++- solar/solar/cli.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33e6f8db..bd7deab6 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,9 @@ sudo docker exec -it foo * To get data for the resource bar (raw and pretty-JSON): ``` solar resource show --tag 'resources/bar' -solar resource show --use-json --tag 'resources/bar' | jq . +solar resource show --json --tag 'resources/bar' | jq . +solar resource show --name 'resource_name' +solar resource show --name 'resource_name' --json | jq . ``` * To clear all resources/connections: diff --git a/solar/solar/cli.py b/solar/solar/cli.py index bf759e22..0958248d 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -306,7 +306,6 @@ def init_cli_resource(): for name, res in sresource.load_all().items(): show = True if kwargs['tag']: - print res.name, res.tags if kwargs['tag'] not in res.tags: show = False if kwargs['name']: From 062efa55957180ca9aba54ef820f117308240583 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 7 Jul 2015 09:05:38 +0200 Subject: [PATCH 4/9] cli.py: better printing during resource/connection creation --- solar/solar/cli.py | 65 +++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/solar/solar/cli.py b/solar/solar/cli.py index 0958248d..dd866444 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -45,6 +45,31 @@ from solar.extensions.modules.discovery import Discovery db = get_db() +# HELPERS +def format_resource_input(resource_name, resource_input_name): + return '{}::{}'.format( + #click.style(resource_name, fg='white', bold=True), + resource_name, + click.style(resource_input_name, fg='yellow') + ) + +def show_emitter_connections(emitter_name, destinations): + inputs = sorted(destinations) + + for emitter_input in inputs: + click.echo( + '{} -> {}'.format( + format_resource_input(emitter_name, emitter_input), + '[{}]'.format( + ', '.join( + format_resource_input(*r) + for r in destinations[emitter_input] + ) + ) + ) + ) + + @click.group() def main(): pass @@ -78,13 +103,6 @@ def assign(resources, nodes): assign_resources_to_nodes(resources, nodes) -# @main.command() -# @click.option('-p', '--profile') -# def connect(profile): -# profile_ = db.get_record('profiles', profile) -# connect_resources(profile_) - - @main.command() def discover(): Discovery({'id': 'discovery'}).discover() @@ -177,12 +195,13 @@ def init_cli_connect(): click.echo('Connect {} to {}'.format(emitter, receiver)) emitter = sresource.load(emitter) receiver = sresource.load(receiver) - click.echo(emitter) - click.echo(receiver) if mapping is not None: mapping = json.loads(mapping) signals.connect(emitter, receiver, mapping=mapping) + clients = signals.Connections.read_clients() + show_emitter_connections(emitter.name, clients[emitter.name]) + @main.command() @click.argument('emitter') @click.argument('receiver') @@ -194,6 +213,9 @@ def init_cli_connect(): click.echo(receiver) signals.disconnect(emitter, receiver) + clients = signals.Connections.read_clients() + show_emitter_connections(emitter.name, clients[emitter.name]) + def init_cli_connections(): @main.group() @@ -207,29 +229,6 @@ def init_cli_connections(): @connections.command() def show(): - def format_resource_input(resource_name, resource_input_name): - return '{}::{}'.format( - #click.style(resource_name, fg='white', bold=True), - resource_name, - click.style(resource_input_name, fg='yellow') - ) - - def show_emitter_connections(emitter_name, destinations): - inputs = sorted(destinations) - - for emitter_input in inputs: - click.echo( - '{} -> {}'.format( - format_resource_input(emitter_name, emitter_input), - '[{}]'.format( - ', '.join( - format_resource_input(*r) - for r in destinations[emitter_input] - ) - ) - ) - ) - clients = signals.Connections.read_clients() keys = sorted(clients) for emitter_name in keys: @@ -293,7 +292,7 @@ def init_cli_resource(): args = json.loads(args) if args else {} resources = vr.create(name, base_path, args) for res in resources: - print res.name + click.echo(res.color_repr()) @resource.command() @click.option('--name', default=None) From 5104edc1577b5159b6997dfcb5e6ad102e7f34a0 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 7 Jul 2015 09:29:22 +0200 Subject: [PATCH 5/9] cli.py: resource create/connect/update alternative input formats --- README.md | 7 ++++--- solar/solar/cli.py | 41 ++++++++++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index bd7deab6..712868d3 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,12 @@ cd /vagrant solar resource create node1 resources/ro_node/ '{"ip":"10.0.0.3", "ssh_key" : "/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key", "ssh_user":"vagrant"}' solar resource create mariadb_service resources/mariadb_service '{"image": "mariadb", "root_password": "mariadb", "port": 3306}' solar resource create keystone_db resources/mariadb_keystone_db/ '{"db_name": "keystone_db", "login_user": "root"}' -solar resource create keystone_db_user resources/mariadb_user/ '{"user_name": "keystone", "user_password": "keystone", "login_user": "root"}' +solar resource create keystone_db_user resources/mariadb_user/ user_name=keystone user_password=keystone login_user=root # another valid format solar connect node1 mariadb_service solar connect node1 keystone_db -solar connect mariadb_service keystone_db --mapping '{"root_password": "login_password", "port": "login_port"}' -solar connect mariadb_service keystone_db_user --mapping '{"root_password": "login_password", "port": "login_port"}' +solar connect mariadb_service keystone_db '{"root_password": "login_password", "port": "login_port"}' +solar connect mariadb_service keystone_db_user 'root_password->login_password port->login_port' # another valid format solar connect keystone_db keystone_db_user solar changes stage @@ -67,6 +67,7 @@ solar changes commit You can fiddle with the above configuration like this: ``` solar resource update keystone_db_user '{"user_password": "new_keystone_password"}' +solar resource update keystone_db_user user_password=new_keystone_password # another valid format solar changes stage solar changes commit diff --git a/solar/solar/cli.py b/solar/solar/cli.py index dd866444..4304b0b7 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -190,14 +190,20 @@ def init_cli_connect(): @main.command() @click.argument('emitter') @click.argument('receiver') - @click.option('--mapping', default=None) + @click.argument('mapping', default='') def connect(mapping, receiver, emitter): + mapping_parsed = {} + click.echo('Connect {} to {}'.format(emitter, receiver)) emitter = sresource.load(emitter) receiver = sresource.load(receiver) - if mapping is not None: - mapping = json.loads(mapping) - signals.connect(emitter, receiver, mapping=mapping) + try: + mapping_parsed.update(json.loads(mapping)) + except ValueError: + for m in mapping.split(): + k, v = m.split('->') + mapping_parsed.update({k: v}) + signals.connect(emitter, receiver, mapping=mapping_parsed) clients = signals.Connections.read_clients() show_emitter_connections(emitter.name, clients[emitter.name]) @@ -286,11 +292,18 @@ def init_cli_resource(): @resource.command() @click.argument('name') @click.argument('base_path', type=click.Path(exists=True, file_okay=False)) - @click.argument('args') + @click.argument('args', nargs=-1) def create(args, base_path, name): + args_parsed = {} + click.echo('create {} {} {}'.format(name, base_path, args)) - args = json.loads(args) if args else {} - resources = vr.create(name, base_path, args) + for arg in args: + try: + args_parsed.update(json.loads(arg)) + except ValueError: + k, v = arg.split('=') + args_parsed.update({k: v}) + resources = vr.create(name, base_path, args_parsed) for res in resources: click.echo(res.color_repr()) @@ -344,13 +357,19 @@ def init_cli_resource(): @resource.command() @click.argument('name') - @click.argument('args') + @click.argument('args', nargs=-1) def update(name, args): - args = json.loads(args) - click.echo('Updating resource {} with args {}'.format(name, args)) + args_parsed = {} + for arg in args: + try: + args_parsed.update(json.loads(arg)) + except ValueError: + k, v = arg.split('=') + args_parsed.update({k: v}) + click.echo('Updating resource {} with args {}'.format(name, args_parsed)) all = sresource.load_all() r = all[name] - r.update(args) + r.update(args_parsed) @resource.command() def validate(): From 497481e71e942a5d7b74df837ca562280d42ae10 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 7 Jul 2015 10:05:39 +0200 Subject: [PATCH 6/9] Validation of resource input value duplication added This is to find inputs whose values are the same and are not mapped between each other. --- README.md | 9 +++- solar/requirements.txt | 1 + solar/solar/cli.py | 16 ++++++- solar/solar/core/virtual_resource.py | 62 ++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 712868d3..722a64be 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,12 @@ cd /vagrant solar resource create node1 resources/ro_node/ '{"ip":"10.0.0.3", "ssh_key" : "/vagrant/.vagrant/machines/solar-dev1/virtualbox/private_key", "ssh_user":"vagrant"}' solar resource create mariadb_service resources/mariadb_service '{"image": "mariadb", "root_password": "mariadb", "port": 3306}' solar resource create keystone_db resources/mariadb_keystone_db/ '{"db_name": "keystone_db", "login_user": "root"}' -solar resource create keystone_db_user resources/mariadb_user/ user_name=keystone user_password=keystone login_user=root # another valid format +solar resource create keystone_db_user resources/mariadb_user/ user_name=keystone user_password=keystone # another valid format solar connect node1 mariadb_service solar connect node1 keystone_db solar connect mariadb_service keystone_db '{"root_password": "login_password", "port": "login_port"}' -solar connect mariadb_service keystone_db_user 'root_password->login_password port->login_port' # another valid format +# solar connect mariadb_service keystone_db_user 'root_password->login_password port->login_port' # another valid format solar connect keystone_db keystone_db_user solar changes stage @@ -79,6 +79,11 @@ solar connections show solar connections graph ``` +* You can make sure that all input values are correct and mapped without duplicating your values with this command: +``` +solar resource validate +``` + # Low level API ## HAProxy deployment (not maintained) diff --git a/solar/requirements.txt b/solar/requirements.txt index 54c43814..69933e6a 100644 --- a/solar/requirements.txt +++ b/solar/requirements.txt @@ -13,3 +13,4 @@ redis==2.10.3 pytest fakeredis Fabric==1.10.2 +tabulate==0.7.5 diff --git a/solar/solar/cli.py b/solar/solar/cli.py index 4304b0b7..68a36b8f 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -24,6 +24,7 @@ import networkx as nx import os import pprint import sys +import tabulate import yaml from solar import utils @@ -375,7 +376,20 @@ def init_cli_resource(): def validate(): errors = vr.validate_resources() for r, error in errors: - print 'ERROR: %s: %s' % (r.name, error) + click.echo('ERROR: %s: %s' % (r.name, error)) + + missing_connections = vr.find_missing_connections() + if missing_connections: + click.echo( + 'The following resources have inputs of the same value ' + 'but are not connected:' + ) + click.echo( + tabulate.tabulate([ + ['%s::%s' % (r1, i1), '%s::%s' % (r2, i2)] + for r1, i1, r2, i2 in missing_connections + ]) + ) @resource.command() @click.argument('path', type=click.Path(exists=True, dir_okay=False)) diff --git a/solar/solar/core/virtual_resource.py b/solar/solar/core/virtual_resource.py index f642c603..6fb109b9 100644 --- a/solar/solar/core/virtual_resource.py +++ b/solar/solar/core/virtual_resource.py @@ -38,6 +38,7 @@ def create_resource(name, base_path, args, virtual_resource=None): resource = resource_module.Resource(name, metadata, args, tags, virtual_resource) return resource + def create_virtual_resource(vr_name, template): resources = template['resources'] connections = [] @@ -63,6 +64,7 @@ def create_virtual_resource(vr_name, template): return created_resources + def create(name, base_path, kwargs, virtual_resource=None): if isinstance(base_path, resource_provider.BaseProvider): base_path = base_path.directory @@ -81,6 +83,7 @@ def create(name, base_path, kwargs, virtual_resource=None): return resources + def validate_resources(): db = resource_module.load_all() all_errors = [] @@ -93,6 +96,61 @@ def validate_resources(): all_errors.append((r, errors)) return all_errors + +def find_inputs_without_source(): + """Find resources and inputs values of which are hardcoded. + + :return: [(resource_name, input_name)] + """ + resources = resource_module.load_all() + + ret = set([(r.name, input_name) for r in resources.values() + for input_name in r.args]) + + clients = signals.Connections.read_clients() + + for dest_dict in clients.values(): + for destinations in dest_dict.values(): + for receiver_name, receiver_input in destinations: + try: + ret.remove((receiver_name, receiver_input)) + except KeyError: + continue + + return list(ret) + + +def find_missing_connections(): + """Find resources whose input values are duplicated + + and they are not connected between each other (i.e. the values + are hardcoded, not coming from connection). + + NOTE: this we could have 2 inputs of the same value living in 2 "circles". + This is not covered, we find only inputs whose value is hardcoded. + + :return: [(resource_name1, input_name1, resource_name2, input_name2)] + """ + ret = set() + + resources = resource_module.load_all() + + inputs_without_source = find_inputs_without_source() + + for resource1, input1 in inputs_without_source: + r1 = resources[resource1] + v1 = r1.args[input1] + + for resource2, input2 in inputs_without_source: + r2 = resources[resource2] + v2 = r2.args[input2] + + if v1 == v2 and resource1 != resource2: + ret.add((resource1, input1, resource2, input2)) + + return list(ret) + + def _compile_file(name, path, kwargs): with open(path) as f: content = f.read() @@ -101,11 +159,13 @@ def _compile_file(name, path, kwargs): template = _get_template(name, content, kwargs, inputs) return template + def get_inputs(content): env = Environment() ast = env.parse(content) return meta.find_undeclared_variables(ast) + def _get_template(name, content, kwargs, inputs): missing = [] for input in inputs: @@ -117,5 +177,7 @@ def _get_template(name, content, kwargs, inputs): template = template.render(str=str, zip=zip, **kwargs) return template + def is_virtual(path): return os.path.isfile(path) + From d6fd52191ee2bfdeaed88f1fcc95fcce0bb534a1 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 7 Jul 2015 10:54:28 +0200 Subject: [PATCH 7/9] Find missing connections: remove duplicated pairs --- solar/solar/core/virtual_resource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solar/solar/core/virtual_resource.py b/solar/solar/core/virtual_resource.py index 6fb109b9..65c9db04 100644 --- a/solar/solar/core/virtual_resource.py +++ b/solar/solar/core/virtual_resource.py @@ -145,7 +145,8 @@ def find_missing_connections(): r2 = resources[resource2] v2 = r2.args[input2] - if v1 == v2 and resource1 != resource2: + if v1 == v2 and resource1 != resource2 and \ + (resource2, input2, resource1, input1) not in ret: ret.add((resource1, input1, resource2, input2)) return list(ret) From 96b88cf80cd6c262b8a253122f0915d1cdbeb752 Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 7 Jul 2015 11:23:10 +0200 Subject: [PATCH 8/9] example.py: keystone service pulled from git, not zip --- example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/example.py b/example.py index 07ce93a3..ba563819 100644 --- a/example.py +++ b/example.py @@ -41,7 +41,8 @@ def deploy(): keystone_db_user = vr.create('keystone_db_user', 'resources/mariadb_user/', {'user_name': 'keystone', 'user_password': 'keystone', 'login_user': 'root'})[0] keystone_config1 = vr.create('keystone_config1', GitProvider(GIT_KEYSTONE_RESOURCE_URL, path='keystone_config'), {'config_dir': '/etc/solar/keystone', 'admin_token': 'admin'})[0] - keystone_service1 = vr.create('keystone_service1', RemoteZipProvider(ZIP_KEYSTONE_RESOURCE_URL, 'keystone_service'), {'port': 5001, 'admin_port': 35357})[0] + #keystone_service1 = vr.create('keystone_service1', RemoteZipProvider(ZIP_KEYSTONE_RESOURCE_URL, 'keystone_service'), {'port': 5001, 'admin_port': 35357})[0] + keystone_service1 = vr.create('keystone_service1', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_service'), {'port': 5001, 'admin_port': 35357})[0] keystone_config2 = vr.create('keystone_config2', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_config'), {'config_dir': '/etc/solar/keystone', 'admin_token': 'admin'})[0] keystone_service2 = vr.create('keystone_service2', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_service'), {'port': 5002, 'admin_port': 35358})[0] From ae9dd8ebd42d15784d92dc8a1a3dfcc0c64d5f7f Mon Sep 17 00:00:00 2001 From: Przemyslaw Kaminski Date: Tue, 7 Jul 2015 11:45:48 +0200 Subject: [PATCH 9/9] cli.py: validate has optional --check-missing-connections --- solar/solar/cli.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/solar/solar/cli.py b/solar/solar/cli.py index 68a36b8f..76e1d99d 100644 --- a/solar/solar/cli.py +++ b/solar/solar/cli.py @@ -373,23 +373,25 @@ def init_cli_resource(): r.update(args_parsed) @resource.command() - def validate(): + @click.option('--check-missing-connections', default=False, is_flag=True) + def validate(check_missing_connections): errors = vr.validate_resources() for r, error in errors: click.echo('ERROR: %s: %s' % (r.name, error)) - missing_connections = vr.find_missing_connections() - if missing_connections: - click.echo( - 'The following resources have inputs of the same value ' - 'but are not connected:' - ) - click.echo( - tabulate.tabulate([ - ['%s::%s' % (r1, i1), '%s::%s' % (r2, i2)] - for r1, i1, r2, i2 in missing_connections - ]) - ) + if check_missing_connections: + missing_connections = vr.find_missing_connections() + if missing_connections: + click.echo( + 'The following resources have inputs of the same value ' + 'but are not connected:' + ) + click.echo( + tabulate.tabulate([ + ['%s::%s' % (r1, i1), '%s::%s' % (r2, i2)] + for r1, i1, r2, i2 in missing_connections + ]) + ) @resource.command() @click.argument('path', type=click.Path(exists=True, dir_okay=False))