From b5ac449ec24385e0bc07612d98f5135212945892 Mon Sep 17 00:00:00 2001 From: Fabrizio Vanni Date: Wed, 2 Sep 2015 18:27:49 +0100 Subject: [PATCH] update type deletion policy in db-init Types with not-matching mappings are not deleted by default. When elasticsearch fails to merge the new mapping with the existing one an error is returned. The type is deleted only when explicitly requested, either interactively or providing the following command-line parameter: --erase Adds the possibility to select the specific mapping to upload with the following command-line parameter: --mapping Change-Id: If18fdba770790d8af03475d45da28c2e40fb7da6 --- README.rst | 56 +++++--- freezer_api/cmd/db_init.py | 98 ++++++++++---- freezer_api/common/db_mappings.py | 206 ++++++++++++++++++------------ tests/test_db_init.py | 40 ++++-- 4 files changed, 265 insertions(+), 135 deletions(-) diff --git a/README.rst b/README.rst index bbb6b137..c90672f7 100644 --- a/README.rst +++ b/README.rst @@ -10,13 +10,7 @@ Freezer API ----------------------------- :: - # pip install keystonemiddleware falcon - -Elasticsearch support -:: - - # pip install elasticsearch - + # pip install -r requirements.txt 1.2 Install freezer_api ----------------------- @@ -25,25 +19,51 @@ Elasticsearch support # git clone https://github.com/stackforge/freezer-api.git # cd freezer-api && sudo python setup.py install -this will install into /usr/local - - 1.3 edit config file -------------------- :: - # sudp cp etc/freezer-api.conf /etc/freezer-api.conf + # sudo cp etc/freezer-api.conf /etc/freezer-api.conf # sudo vi /etc/freezer-api.conf -1.4 run simple instance +1.4 setup/configure the db +-------------------------- +The currently supported db is Elasticsearch. In case you are using a dedicated instance +of the server, you'll need to start it. Depending on the OS flavor it might be a: +:: + + # service elasticsearch start + +or, on systemd:: + + # systemctl start elasticsearch + +Elasticsearch needs to know what type of data each document's field contains. +This information is contained in the "mapping", or schema definition. +Elasticsearch will use dynamic mapping to try to guess the field type from +the basic datatypes available in JSON, but some field's properties have to be +explicitly declared to tune the indexing engine. +To do that, use the freezer-db-init command: +:: + + # freezer-db-init [db-host] + +The url of the db-host is optional and can be automatically guessed from +/etc/freezer-api.conf + +To get information about optional additional parameters: +:: + + freezer-db-init -h + +1.5 run simple instance ----------------------- :: # freezer-api - -1.5 examples running using uwsgi +1.6 examples running using uwsgi -------------------------------- :: @@ -70,13 +90,13 @@ backups which share the same container,hostname and backupname =================== :: - keystone user-create --name freezer --pass FREEZER_PWD - keystone user-role-add --user freezer --tenant service --role admin + # keystone user-create --name freezer --pass FREEZER_PWD + # keystone user-role-add --user freezer --tenant service --role admin - keystone service-create --name freezer --type backup \ + # keystone service-create --name freezer --type backup \ --description "Freezer Backup Service" - keystone endpoint-create \ + # keystone endpoint-create \ --service-id $(keystone service-list | awk '/ backup / {print $2}') \ --publicurl http://freezer_api_publicurl:port \ --internalurl http://freezer_api_internalurl:port \ diff --git a/freezer_api/cmd/db_init.py b/freezer_api/cmd/db_init.py index c4214d78..091ec3eb 100755 --- a/freezer_api/cmd/db_init.py +++ b/freezer_api/cmd/db_init.py @@ -37,16 +37,19 @@ DEFAULT_ES_SERVER_PORT = 9200 DEFAULT_INDEX = 'freezer' +class MergeMappingException(Exception): + pass + + class ElastichsearchEngine(object): - def __init__(self, es_url, es_index, test_only, always_yes, verbose): + def __init__(self, es_url, es_index, args): self.es_url = es_url self.es_index = es_index - self.test_only = test_only - self.always_yes = always_yes - self.verbose = verbose + self.args = args + self.exit_code = os.EX_OK - def verbose_print(self, message): - if self.verbose: + def verbose_print(self, message, level=1): + if self.args.verbose >= level: print(message) def put_mappings(self, mappings): @@ -56,6 +59,7 @@ class ElastichsearchEngine(object): print '{0}/{1} MATCHES'.format(self.es_index, es_type) else: self.askput_mapping(es_type, mapping) + return self.exit_code def check_index_exists(self): url = '{0}/{1}'.format(self.es_url, self.es_index) @@ -80,16 +84,47 @@ class ElastichsearchEngine(object): return mapping == current_mappings.get(es_type, {}) def askput_mapping(self, es_type, mapping): - if self.test_only: + if self.args.test_only: print '{0}/{1} DOES NOT MATCH'.format(self.es_index, es_type) + self.exit_code = os.EX_DATAERR return - prompt_message = ('{0}/{1}/{2} needs to be deleted. ' + prompt_message = ('{0}/{1}/{2} needs to be updated. ' + 'Proceed ? (y/n)' + .format(self.es_url, + self.es_index, + es_type)) + if not self.proceed(prompt_message, self.args.yes): + return + + self.verbose_print('Trying to upload mappings ...') + try: + self.put_mapping(es_type, mapping) + except MergeMappingException as e: + self.verbose_print('Unable to merge mappings.') + self.verbose_print(e, 2) + else: + print "Mappings updated" + return + + if self.args.yes and not self.args.erase: + # explicit consent to update without explicit consent to erase: + # do not erase type and return error code + self.exit_code = os.EX_DATAERR + print ('{0}/{1} DOES NOT MATCH. ' + 'Need explicit consent to erase types' + .format(self.es_index, es_type)) + return + prompt_message = ('Type {0}/{1}/{2} needs to be deleted. ' 'Proceed (y/n) ? '.format(self.es_url, self.es_index, es_type)) - if self.always_yes or self.proceed(prompt_message): - self.delete_type(es_type) - self.put_mapping(es_type, mapping) + if not self.proceed(prompt_message, self.args.erase): + return + + self.verbose_print('Deleting type {0}'.format(es_type)) + self.delete_type(es_type) + self.verbose_print('Uploading mappings ...') + self.put_mapping(es_type, mapping) def delete_type(self, es_type): url = '{0}/{1}/{2}'.format(self.es_url, self.es_index, es_type) @@ -110,11 +145,11 @@ class ElastichsearchEngine(object): if r.status_code == requests.codes.OK: print "Type {0} mapping created".format(url) else: - raise Exception('Type mapping creation error {0}: ' - '{1}'.format(r.status_code, r.text)) + raise MergeMappingException('Type mapping creation error {0}: ' + '{1}'.format(r.status_code, r.text)) - def proceed(self, message): - if self.always_yes: + def proceed(self, message, assume_yes=False): + if assume_yes: return True while True: selection = raw_input(message) @@ -124,7 +159,7 @@ class ElastichsearchEngine(object): return False -def get_args(): +def get_args(mapping_choices): arg_parser = argparse.ArgumentParser() arg_parser.add_argument( 'host', action='store', default='', nargs='?', @@ -134,16 +169,27 @@ def get_args(): help=('The DB server port ' '(default: {0})'.format(DEFAULT_ES_SERVER_PORT)), dest='port', default=0) + arg_parser.add_argument( + '-m', '--mapping', action='store', + help=('Specific mapping to upload. Valid choices: {0}' + .format(','.join(mapping_choices))), + choices=mapping_choices, + dest='select_mapping', default='') arg_parser.add_argument( '-i', '--index', action='store', help='The DB index (default "{0}")'.format(DEFAULT_INDEX), dest='index') arg_parser.add_argument( '-y', '--yes', action='store_true', - help="Automatic confirmation to index removal", + help="Automatic confirmation to mapping update", dest='yes', default=False) arg_parser.add_argument( - '-v', '--verbose', action='store_true', + '-e', '--erase', action='store_true', + help=("Enable index deletion in case mapping update " + "fails due to incompatible changes"), + dest='erase', default=False) + arg_parser.add_argument( + '-v', '--verbose', action='count', help="Verbose", dest='verbose', default=False) arg_parser.add_argument( @@ -241,29 +287,29 @@ def get_db_params(args): def main(): - args = get_args() + mappings = db_mappings.get_mappings() + + args = get_args(mapping_choices=mappings.keys()) elasticsearch_url, elasticsearch_index = get_db_params(args) es_manager = ElastichsearchEngine(es_url=elasticsearch_url, es_index=elasticsearch_index, - test_only=args.test_only, - always_yes=args.yes, - verbose=args.verbose) - + args=args) if args.verbose: print " db url: {0}".format(elasticsearch_url) print "db index: {0}".format(elasticsearch_index) - mappings = db_mappings.get_mappings() + if args.select_mapping: + mappings = {args.select_mapping: mappings[args.select_mapping]} try: - es_manager.put_mappings(mappings) + exit_code = es_manager.put_mappings(mappings) except Exception as e: print "ERROR {0}".format(e) return os.EX_DATAERR - return os.EX_OK + return exit_code if __name__ == '__main__': sys.exit(main()) diff --git a/freezer_api/common/db_mappings.py b/freezer_api/common/db_mappings.py index 548dbe0b..b069c661 100644 --- a/freezer_api/common/db_mappings.py +++ b/freezer_api/common/db_mappings.py @@ -21,121 +21,161 @@ Hudson (tjh@cryptsoft.com). clients_mapping = { - u'properties': { - u'client': { - u'properties': { - u'client_id': { - u'index': u'not_analyzed', - u'type': u'string', + "properties": { + "client": { + "properties": { + "client_id": { + "index": "not_analyzed", + "type": "string", }, - u'config_id': { - u'index': u'not_analyzed', - u'type': u'string', + "config_id": { + "index": "not_analyzed", + "type": "string", }, - u'description': { - u'type': u'string', + "description": { + "type": "string", }, - u'hostname': { - u'type': u'string', + "hostname": { + "type": "string", }, }, }, - u'user_id': { - u'index': u'not_analyzed', - u'type': u'string', + "user_id": { + "index": "not_analyzed", + "type": "string", + }, + "uuid": { + "index": "not_analyzed", + "type": "string" }, }, } backups_mapping = { - u'properties': { - u'backup_id': { - u'index': u'not_analyzed', - u'type': u'string', + "properties": { + "backup_id": { + "index": "not_analyzed", + "type": "string", }, - u'backup_metadata': { - u'properties': { - u'backup_name': { - u'index': u'not_analyzed', - u'type': u'string', + "backup_metadata": { + "properties": { + "action": { + "type": "string", }, - u'backup_session': { - u'type': u'long', + "always_level": { + "type": "boolean", }, - u'backup_size_compressed': { - u'type': u'long', + "backup_media": { + "type": "string", }, - u'backup_size_uncompressed': { - u'type': u'long', + "backup_name": { + "index": "not_analyzed", + "type": "string", }, - u'broken_links': { - u'index': u'not_analyzed', - u'type': u'string', + "backup_session": { + "type": "long", }, - u'cli': { - u'type': u'string', + "backup_size_compressed": { + "type": "long", }, - u'client_os': { - u'type': u'string', + "backup_size_uncompressed": { + "type": "long", }, - u'compression_alg': { - u'type': u'string', + "broken_links": { + "index": "not_analyzed", + "type": "string", }, - u'container': { - u'index': u'not_analyzed', - u'type': u'string', + "cli": { + "type": "string", }, - u'encrypted': { - u'type': u'boolean', + "client_os": { + "type": "string", }, - u'excluded_files': { - u'type': u'string', + "client_version": { + "type": "string", }, - u'fs_real_path': { - u'type': u'string', + "compression_alg": { + "type": "string", }, - u'host_name': { - u'index': u'not_analyzed', - u'type': u'string', + "container": { + "index": "not_analyzed", + "type": "string", }, - u'level': { - u'type': u'long', + "container_segments": { + "type": "string", }, - u'max_level': { - u'type': u'long', + "curr_backup_level": { + "type": "string", }, - u'mode': { - u'type': u'string', + "current_level": { + "type": "string", }, - u'timestamp': { - u'type': u'long', + "dry_run": { + "type": "boolean", }, - u'total_backup_session_size': { - u'type': u'long', + "encrypted": { + "type": "boolean", }, - u'total_broken_links': { - u'type': u'long', + "excluded_files": { + "type": "string", }, - u'total_directories': { - u'type': u'long', + "fs_real_path": { + "type": "string", }, - u'total_fs_files': { - u'type': u'long', + "host_name": { + "index": "not_analyzed", + "type": "string", }, - u'version': { - u'type': u'string', + "hostname": { + "type": "string", }, - u'vol_snap_path': { - u'type': u'string', + "level": { + "type": "long", + }, + "max_level": { + "type": "long", + }, + "meta_data_file": { + "type": "string", + }, + "mode": { + "type": "string", + }, + "path_to_backup": { + "type": "string", + }, + "time_stamp": { + "type": "string", + }, + "timestamp": { + "type": "long", + }, + "total_backup_session_size": { + "type": "long", + }, + "total_broken_links": { + "type": "long", + }, + "total_directories": { + "type": "long", + }, + "total_fs_files": { + "type": "long", + }, + "version": { + "type": "string", + }, + "vol_snap_path": { + "type": "string", }, }, }, - u'user_id': { - u'index': u'not_analyzed', - u'type': u'string', + "user_id": { + "index": "not_analyzed", + "type": "string", }, - u'user_name': { - u'type': u'string', + "user_name": { + "type": "string", }, }, } @@ -165,6 +205,9 @@ jobs_mapping = { "dry_run": { "type": "boolean" }, + "log_file": { + "type": "string" + }, "lvm_auto_snap": { "type": "string" }, @@ -203,6 +246,9 @@ jobs_mapping = { }, "restore_abs_path": { "type": "string" + }, + "restore_from_host": { + "type": "string" } } }, @@ -279,7 +325,7 @@ jobs_mapping = { def get_mappings(): return { - u'jobs': jobs_mapping, - u'backups': backups_mapping, - u'clients': clients_mapping + "jobs": jobs_mapping, + "backups": backups_mapping, + "clients": clients_mapping } diff --git a/tests/test_db_init.py b/tests/test_db_init.py index 261b7741..4b6bdfb9 100644 --- a/tests/test_db_init.py +++ b/tests/test_db_init.py @@ -31,7 +31,8 @@ from freezer_api.cmd.db_init import (ElastichsearchEngine, parse_config_file, get_db_params, main, - DEFAULT_CONF_PATH) + DEFAULT_CONF_PATH, + MergeMappingException) from freezer_api.common import db_mappings @@ -45,11 +46,15 @@ class TestElasticsearchEngine(unittest.TestCase): } self.mock_resp = Mock() + self.mock_args = Mock() + self.mock_args.test_only = False + self.mock_args.always_yes = False + self.mock_args.verbose = 1 + self.mock_args.select_mapping = '' + self.mock_args.erase = False self.es_manager = ElastichsearchEngine(es_url='http://test:9333', es_index='freezerindex', - test_only=False, - always_yes=False, - verbose=True) + args=self.mock_args) def test_new(self): self.assertIsInstance(self.es_manager, ElastichsearchEngine) @@ -72,17 +77,19 @@ class TestElasticsearchEngine(unittest.TestCase): @patch.object(ElastichsearchEngine, 'proceed') @patch.object(ElastichsearchEngine, 'delete_type') @patch.object(ElastichsearchEngine, 'put_mapping') - def test_askput_calls_delete_and_put_mappunts_when_always_yes(self, + def test_askput_calls_delete_and_put_mappings_when_always_yes_and_erase(self, mock_put_mapping, mock_delete_type, mock_proceed): - self.es_manager.always_yes = True + self.mock_args.yes = True + self.mock_args.erase = True + mock_put_mapping.side_effect = [MergeMappingException('regular test failure'), 0] res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs']) + self.assertTrue(mock_put_mapping.called) mock_delete_type.assert_called_once_with('jobs') - mock_put_mapping.assert_called_once_with('jobs', self.test_mappings['jobs']) def test_askput_does_nothing_when_test_only(self): - self.es_manager.test_only = True + self.mock_args.test_only = True res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs']) self.assertEquals(None, res) @@ -176,8 +183,7 @@ class TestElasticsearchEngine(unittest.TestCase): _raw_input.assert_called_once_with('are you drunk ?') def test_proceed_returns_true_when_always_yes(self): - self.es_manager.always_yes = True - res = self.es_manager.proceed('ask me not') + res = self.es_manager.proceed('ask me not', True) self.assertTrue(res) @patch('freezer_api.cmd.db_init.requests') @@ -210,12 +216,20 @@ class TestElasticsearchEngine(unittest.TestCase): class TestDbInit(unittest.TestCase): + def setUp(self): + self.mock_args = Mock() + self.mock_args.test_only = False + self.mock_args.always_yes = False + self.mock_args.verbose = 1 + self.mock_args.select_mapping = '' + self.mock_args.erase = False + @patch('freezer_api.cmd.db_init.argparse.ArgumentParser') def test_get_args_calls_add_argument(self, mock_ArgumentParser): mock_arg_parser = Mock() mock_ArgumentParser.return_value = mock_arg_parser - retval = get_args() + retval = get_args([]) call_count = mock_arg_parser.add_argument.call_count self.assertGreater(call_count, 6) @@ -262,8 +276,11 @@ class TestDbInit(unittest.TestCase): @patch('freezer_api.cmd.db_init.get_args') def test_main_calls_esmanager_put_mappings_with_mappings(self, mock_get_args, mock_get_db_params, mock_ElastichsearchEngine): + mock_get_args.return_value = self.mock_args mock_get_db_params.return_value = Mock(), Mock() mock_es_manager = Mock() + mock_es_manager.put_mappings.return_value = os.EX_OK + mock_ElastichsearchEngine.return_value = mock_es_manager res = main() @@ -276,6 +293,7 @@ class TestDbInit(unittest.TestCase): @patch('freezer_api.cmd.db_init.get_args') def test_main_return_EX_DATAERR_exitcode_on_error(self, mock_get_args, mock_get_db_params, mock_ElastichsearchEngine): + mock_get_args.return_value = self.mock_args mock_get_db_params.return_value = Mock(), Mock() mock_es_manager = Mock() mock_ElastichsearchEngine.return_value = mock_es_manager