diff --git a/.gitignore b/.gitignore index f81252a5..000e943f 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ vagrant-settings.yaml .ssh/ .cache + +.tox diff --git a/docs/removal.md b/docs/removal.md new file mode 100644 index 00000000..d932077a --- /dev/null +++ b/docs/removal.md @@ -0,0 +1,29 @@ +# Problems to solve with removal operation + +1. It is tricky to figure out what to do with data that will be left when +you are removing resource that is a parent for other resources. + +The basic example is a node resource. +If hosts_file1 subscribed to node properties, and we will just remove +node - hosts_file1 will be left with corrupted data. +Validation is not a solution, because we can not expect user to remove +each resource one-by-one. + +log task=hosts_file1.run uid=c1545041-a5c5-400e-8c46-ad52d871e6c3 + ++ ip: None + ++ ssh_user: None + ++ hosts: [{u'ip': None, u'name': u'riak_server1.solar'}] + ++ ssh_key: None + +Proposed solution: + +Add `solar res remove node1 -r` where *r* stands for recursive. +During this operation we will find all childs of specified resource, and +stage them for removal as well. + +2. If so we need to be able to determine what to do with child resource +on removal. +Basically this seems like another type of event: +hosts1.remove -> success -> node1.remove +And +hosts2.update -> success -> node2.remove diff --git a/solar/solar/cli/main.py b/solar/solar/cli/main.py index 569ce1d4..e7231ecb 100644 --- a/solar/solar/cli/main.py +++ b/solar/solar/cli/main.py @@ -41,6 +41,7 @@ from solar.cli import executors from solar.cli.orch import orchestration from solar.cli.system_log import changes from solar.cli.events import events +from solar.cli.resource import resource as cli_resource # HELPERS @@ -157,203 +158,12 @@ def init_cli_connections(): fabric_api.local('dot -Tsvg graph.dot -o graph.svg') -def init_cli_resource(): - @main.group() - def resource(): - pass - - @resource.command() - @click.argument('action') - @click.argument('resource') - @click.option('-d', '--dry-run', default=False, is_flag=True) - @click.option('-m', '--dry-run-mapping', default='{}') - def action(dry_run_mapping, dry_run, action, resource): - if dry_run: - dry_run_executor = executors.DryRunExecutor(mapping=json.loads(dry_run_mapping)) - - click.echo( - 'action {} for resource {}'.format(action, resource) - ) - - r = sresource.load(resource) - try: - actions.resource_action(r, action) - except errors.SolarError as e: - log.debug(e) - sys.exit(1) - - if dry_run: - click.echo('EXECUTED:') - for key in dry_run_executor.executed: - click.echo('{}: {}'.format( - click.style(dry_run_executor.compute_hash(key), fg='green'), - str(key) - )) - - @resource.command() - @click.argument('resource') - def backtrack_inputs(resource): - r = sresource.load(resource) - - inputs = [] - - def backtrack(i): - def format_input(i): - return '{}::{}'.format(i.resource.name, i.name) - - if isinstance(i, list): - return [backtrack(bi) for bi in i] - - if isinstance(i, dict): - return { - k: backtrack(bi) for k, bi in i.items() - } - - bi = i.backtrack_value_emitter(level=1) - if isinstance(i, orm.DBResourceInput) and isinstance(bi, orm.DBResourceInput) and i == bi: - return (format_input(i), ) - - return (format_input(i), backtrack(bi)) - - for i in r.resource_inputs().values(): - click.echo(yaml.safe_dump({i.name: backtrack(i)}, default_flow_style=False)) - - @resource.command() - def compile_all(): - from solar.core.resource import compiler - - destination_path = utils.read_config()['resources-compiled-file'] - - if os.path.exists(destination_path): - os.remove(destination_path) - - for path in utils.find_by_mask(utils.read_config()['resources-files-mask']): - meta = utils.yaml_load(path) - meta['base_path'] = os.path.dirname(path) - - compiler.compile(meta) - - @resource.command() - def clear_all(): - click.echo('Clearing all resources and connections') - orm.db.clear() - - @resource.command() - @click.argument('name') - @click.argument( - 'base_path', type=click.Path(exists=True, resolve_path=True)) - @click.argument('args', nargs=-1) - def create(args, base_path, name): - args_parsed = {} - - click.echo('create {} {} {}'.format(name, base_path, args)) - for arg in args: - try: - args_parsed.update(json.loads(arg)) - except ValueError: - k, v = arg.split('=') - args_parsed.update({k: v}) - resources = vr.create(name, base_path, args=args_parsed) - for res in resources: - click.echo(res.color_repr()) - - @resource.command() - @click.option('--name', default=None) - @click.option('--tag', default=None) - @click.option('--json', default=False, is_flag=True) - @click.option('--color', default=True, is_flag=True) - def show(**kwargs): - resources = [] - - for res in sresource.load_all(): - show = True - if kwargs['tag']: - if kwargs['tag'] not in res.tags: - show = False - if kwargs['name']: - if res.name != kwargs['name']: - show = False - - if show: - resources.append(res) - - echo = click.echo_via_pager - if kwargs['json']: - output = json.dumps([r.to_dict() for r in resources], indent=2) - echo = click.echo - else: - if kwargs['color']: - formatter = lambda r: r.color_repr() - else: - formatter = lambda r: unicode(r) - output = '\n'.join(formatter(r) for r in resources) - - if output: - echo(output) - - @resource.command() - @click.argument('resource_name') - @click.argument('tag_name') - @click.option('--add/--delete', default=True) - def tag(add, tag_name, resource_name): - click.echo('Tag {} with {} {}'.format(resource_name, tag_name, add)) - r = sresource.load(resource_name) - if add: - r.add_tag(tag_name) - else: - r.remove_tag(tag_name) - # TODO: the above functions should save resource automatically to the DB - - @resource.command() - @click.argument('name') - @click.argument('args', nargs=-1) - def update(name, args): - args_parsed = {} - for arg in args: - try: - args_parsed.update(json.loads(arg)) - except ValueError: - k, v = arg.split('=') - args_parsed.update({k: v}) - click.echo('Updating resource {} with args {}'.format(name, args_parsed)) - res = sresource.load(name) - res.update(args_parsed) - - @resource.command() - @click.option('--check-missing-connections', default=False, is_flag=True) - def validate(check_missing_connections): - errors = vr.validate_resources() - for r, error in errors: - click.echo('ERROR: %s: %s' % (r.name, error)) - - if check_missing_connections: - missing_connections = vr.find_missing_connections() - if missing_connections: - click.echo( - 'The following resources have inputs of the same value ' - 'but are not connected:' - ) - click.echo( - tabulate.tabulate([ - ['%s::%s' % (r1, i1), '%s::%s' % (r2, i2)] - for r1, i1, r2, i2 in missing_connections - ]) - ) - - @resource.command() - @click.argument('path', type=click.Path(exists=True, dir_okay=False)) - def get_inputs(path): - with open(path) as f: - content = f.read() - click.echo(vr.get_inputs(content)) - - def run(): init_actions() init_cli_connect() init_cli_connections() - init_cli_resource() + main.add_command(cli_resource) main.add_command(orchestration) main.add_command(changes) main.add_command(events) diff --git a/solar/solar/cli/resource.py b/solar/solar/cli/resource.py new file mode 100644 index 00000000..d4938a69 --- /dev/null +++ b/solar/solar/cli/resource.py @@ -0,0 +1,227 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys +import os +import json +import yaml +import tabulate + +import click + +from solar.core import actions +from solar.core import resource as sresource +from solar.core.resource import virtual_resource as vr +from solar.core.log import log +from solar import errors +from solar.interfaces import orm +from solar import utils + +from solar.cli import executors + + +@click.group() +def resource(): + pass + +@resource.command() +@click.argument('action') +@click.argument('resource') +@click.option('-d', '--dry-run', default=False, is_flag=True) +@click.option('-m', '--dry-run-mapping', default='{}') +def action(dry_run_mapping, dry_run, action, resource): + if dry_run: + dry_run_executor = executors.DryRunExecutor(mapping=json.loads(dry_run_mapping)) + + click.echo( + 'action {} for resource {}'.format(action, resource) + ) + + r = sresource.load(resource) + try: + actions.resource_action(r, action) + except errors.SolarError as e: + log.debug(e) + sys.exit(1) + + if dry_run: + click.echo('EXECUTED:') + for key in dry_run_executor.executed: + click.echo('{}: {}'.format( + click.style(dry_run_executor.compute_hash(key), fg='green'), + str(key) + )) + +@resource.command() +@click.argument('resource') +def backtrack_inputs(resource): + r = sresource.load(resource) + + inputs = [] + + def backtrack(i): + def format_input(i): + return '{}::{}'.format(i.resource.name, i.name) + + if isinstance(i, list): + return [backtrack(bi) for bi in i] + + if isinstance(i, dict): + return { + k: backtrack(bi) for k, bi in i.items() + } + + bi = i.backtrack_value_emitter(level=1) + if isinstance(i, orm.DBResourceInput) and isinstance(bi, orm.DBResourceInput) and i == bi: + return (format_input(i), ) + + return (format_input(i), backtrack(bi)) + + for i in r.resource_inputs().values(): + click.echo(yaml.safe_dump({i.name: backtrack(i)}, default_flow_style=False)) + +@resource.command() +def compile_all(): + from solar.core.resource import compiler + + destination_path = utils.read_config()['resources-compiled-file'] + + if os.path.exists(destination_path): + os.remove(destination_path) + + for path in utils.find_by_mask(utils.read_config()['resources-files-mask']): + meta = utils.yaml_load(path) + meta['base_path'] = os.path.dirname(path) + + compiler.compile(meta) + +@resource.command() +def clear_all(): + click.echo('Clearing all resources and connections') + orm.db.clear() + +@resource.command() +@click.argument('name') +@click.argument( + 'base_path', type=click.Path(exists=True, resolve_path=True)) +@click.argument('args', nargs=-1) +def create(args, base_path, name): + args_parsed = {} + + click.echo('create {} {} {}'.format(name, base_path, args)) + for arg in args: + try: + args_parsed.update(json.loads(arg)) + except ValueError: + k, v = arg.split('=') + args_parsed.update({k: v}) + resources = vr.create(name, base_path, args=args_parsed) + for res in resources: + click.echo(res.color_repr()) + +@resource.command() +@click.option('--name', default=None) +@click.option('--tag', default=None) +@click.option('--json', default=False, is_flag=True) +@click.option('--color', default=True, is_flag=True) +def show(**kwargs): + resources = [] + + for res in sresource.load_all(): + show = True + if kwargs['tag']: + if kwargs['tag'] not in res.tags: + show = False + if kwargs['name']: + if res.name != kwargs['name']: + show = False + + if show: + resources.append(res) + + echo = click.echo_via_pager + if kwargs['json']: + output = json.dumps([r.to_dict() for r in resources], indent=2) + echo = click.echo + else: + if kwargs['color']: + formatter = lambda r: r.color_repr() + else: + formatter = lambda r: unicode(r) + output = '\n'.join(formatter(r) for r in resources) + + if output: + echo(output) + +@resource.command() +@click.argument('resource_name') +@click.argument('tag_name') +@click.option('--add/--delete', default=True) +def tag(add, tag_name, resource_name): + click.echo('Tag {} with {} {}'.format(resource_name, tag_name, add)) + r = sresource.load(resource_name) + if add: + r.add_tag(tag_name) + else: + r.remove_tag(tag_name) + # TODO: the above functions should save resource automatically to the DB + +@resource.command() +@click.argument('name') +@click.argument('args', nargs=-1) +def update(name, args): + args_parsed = {} + for arg in args: + try: + args_parsed.update(json.loads(arg)) + except ValueError: + k, v = arg.split('=') + args_parsed.update({k: v}) + click.echo('Updating resource {} with args {}'.format(name, args_parsed)) + res = sresource.load(name) + res.update(args_parsed) + +@resource.command() +@click.option('--check-missing-connections', default=False, is_flag=True) +def validate(check_missing_connections): + errors = vr.validate_resources() + for r, error in errors: + click.echo('ERROR: %s: %s' % (r.name, error)) + + if check_missing_connections: + missing_connections = vr.find_missing_connections() + if missing_connections: + click.echo( + 'The following resources have inputs of the same value ' + 'but are not connected:' + ) + click.echo( + tabulate.tabulate([ + ['%s::%s' % (r1, i1), '%s::%s' % (r2, i2)] + for r1, i1, r2, i2 in missing_connections + ]) + ) + +@resource.command() +@click.argument('path', type=click.Path(exists=True, dir_okay=False)) +def get_inputs(path): + with open(path) as f: + content = f.read() + click.echo(vr.get_inputs(content)) + +@resource.command() +@click.argument('name') +def remove(name): + res = sresource.load(name) + res.delete() diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py index 00b57b8a..bb07663a 100644 --- a/solar/solar/core/resource/resource.py +++ b/solar/solar/core/resource/resource.py @@ -112,6 +112,9 @@ class Resource(object): i.value = v i.save() + def delete(self): + return self.db_obj.delete() + def resource_inputs(self): return { i.name: i for i in self.db_obj.inputs.as_set() diff --git a/solar/solar/interfaces/orm.py b/solar/solar/interfaces/orm.py index 4f3e6a51..3cb69113 100644 --- a/solar/solar/interfaces/orm.py +++ b/solar/solar/interfaces/orm.py @@ -413,6 +413,17 @@ class DBResourceInput(DBObject): )[0].start_node.properties ) + def delete(self): + db.delete_relations( + source=self._db_node, + type_=base.BaseGraphDB.RELATION_TYPES.input_to_input + ) + db.delete_relations( + dest=self._db_node, + type_=base.BaseGraphDB.RELATION_TYPES.input_to_input + ) + super(DBResourceInput, self).delete() + def backtrack_value_emitter(self, level=None): # TODO: this is actually just fetching head element in linked list # so this whole algorithm can be moved to the db backend probably @@ -559,6 +570,12 @@ class DBResource(DBObject): event.save() self.events.add(event) + def delete(self): + for input in self.inputs.as_set(): + self.inputs.remove(input) + input.delete() + super(DBResource, self).delete() + # TODO: remove this if __name__ == '__main__': diff --git a/solar/solar/system_log/change.py b/solar/solar/system_log/change.py index 110bb486..b9d6cc06 100644 --- a/solar/solar/system_log/change.py +++ b/solar/solar/system_log/change.py @@ -52,7 +52,8 @@ def create_logitem(resource, action, diffed): def _stage_changes(staged_resources, commited_resources, staged_log): - for res_uid in staged_resources.keys(): + union = set(staged_resources.keys()) | set(commited_resources.keys()) + for res_uid in union: commited_data = commited_resources.get(res_uid, {}) staged_data = staged_resources.get(res_uid, {}) diff --git a/solar/solar/test/test_orm.py b/solar/solar/test/test_orm.py index 43ad6a20..38df33a6 100644 --- a/solar/solar/test/test_orm.py +++ b/solar/solar/test/test_orm.py @@ -228,6 +228,12 @@ class TestResourceORM(BaseResourceTest): r.add_input('ip', 'str!', '10.0.0.2') self.assertEqual(len(r.inputs.as_set()), 1) + def test_delete_resource(self): + r = orm.DBResource(id='test1', name='test1', base_path='x') + r.save() + + r.add_input('ip', 'str!', '10.0.0.2') + class TestResourceInputORM(BaseResourceTest): def test_backtrack_simple(self): diff --git a/solar/solar/test/test_resource.py b/solar/solar/test/test_resource.py index 20171412..a3af0256 100644 --- a/solar/solar/test/test_resource.py +++ b/solar/solar/test/test_resource.py @@ -93,3 +93,29 @@ input: self.assertDictEqual(sample.args, sample_l.args) self.assertListEqual(sample.tags, sample_l.tags) + + def test_removal(self): + """Test that connection removed with resource.""" + sample_meta_dir = self.make_resource_meta(""" +id: sample +handler: ansible +version: 1.0.0 +input: + value: + schema: int + value: 0 + """) + + sample1 = self.create_resource( + 'sample1', sample_meta_dir, {'value': 1} + ) + sample2 = self.create_resource( + 'sample2', sample_meta_dir, {} + ) + signals.connect(sample1, sample2) + self.assertEqual(sample1.args['value'], sample2.args['value']) + + sample1 = resource.load('sample1') + sample2 = resource.load('sample2') + sample1.delete() + self.assertEqual(sample2.args['value'], 0) diff --git a/solar/test-requirements.txt b/solar/test-requirements.txt index 49dcbbbb..18ecb06e 100644 --- a/solar/test-requirements.txt +++ b/solar/test-requirements.txt @@ -1,2 +1,4 @@ -r requirements.txt pytest-mock +tox +hacking==0.7 diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..b2795f6f --- /dev/null +++ b/tox.ini @@ -0,0 +1,37 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = py27,pep8 + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/test-requirements.txt +commands = + py.test {posargs:solar/solar/test} + +[testenv:pep8] +deps = hacking==0.7 +usedevelop = False +commands = + flake8 {posargs:solar/solar} + + +[testenv:venv] +deps = -r{toxinidir}/requirements.txt +commands = {posargs:} + +[testenv:devenv] +envdir = devenv +usedevelop = True + +[flake8] +# NOTE(eli): H304 is "No relative imports" error, relative +# imports are required for extensions which can be moved +# from nailgun directory to different place +ignore = H234,H302,H802,H304 +exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs +show-pep8 = True +show-source = True +count = True