diff --git a/README.rst b/README.rst index bbb6b137..c90672f7 100644 --- a/README.rst +++ b/README.rst @@ -10,13 +10,7 @@ Freezer API ----------------------------- :: - # pip install keystonemiddleware falcon - -Elasticsearch support -:: - - # pip install elasticsearch - + # pip install -r requirements.txt 1.2 Install freezer_api ----------------------- @@ -25,25 +19,51 @@ Elasticsearch support # git clone https://github.com/stackforge/freezer-api.git # cd freezer-api && sudo python setup.py install -this will install into /usr/local - - 1.3 edit config file -------------------- :: - # sudp cp etc/freezer-api.conf /etc/freezer-api.conf + # sudo cp etc/freezer-api.conf /etc/freezer-api.conf # sudo vi /etc/freezer-api.conf -1.4 run simple instance +1.4 setup/configure the db +-------------------------- +The currently supported db is Elasticsearch. In case you are using a dedicated instance +of the server, you'll need to start it. Depending on the OS flavor it might be a: +:: + + # service elasticsearch start + +or, on systemd:: + + # systemctl start elasticsearch + +Elasticsearch needs to know what type of data each document's field contains. +This information is contained in the "mapping", or schema definition. +Elasticsearch will use dynamic mapping to try to guess the field type from +the basic datatypes available in JSON, but some field's properties have to be +explicitly declared to tune the indexing engine. +To do that, use the freezer-db-init command: +:: + + # freezer-db-init [db-host] + +The url of the db-host is optional and can be automatically guessed from +/etc/freezer-api.conf + +To get information about optional additional parameters: +:: + + freezer-db-init -h + +1.5 run simple instance ----------------------- :: # freezer-api - -1.5 examples running using uwsgi +1.6 examples running using uwsgi -------------------------------- :: @@ -70,13 +90,13 @@ backups which share the same container,hostname and backupname =================== :: - keystone user-create --name freezer --pass FREEZER_PWD - keystone user-role-add --user freezer --tenant service --role admin + # keystone user-create --name freezer --pass FREEZER_PWD + # keystone user-role-add --user freezer --tenant service --role admin - keystone service-create --name freezer --type backup \ + # keystone service-create --name freezer --type backup \ --description "Freezer Backup Service" - keystone endpoint-create \ + # keystone endpoint-create \ --service-id $(keystone service-list | awk '/ backup / {print $2}') \ --publicurl http://freezer_api_publicurl:port \ --internalurl http://freezer_api_internalurl:port \ diff --git a/freezer_api/cmd/db_init.py b/freezer_api/cmd/db_init.py index c4214d78..091ec3eb 100755 --- a/freezer_api/cmd/db_init.py +++ b/freezer_api/cmd/db_init.py @@ -37,16 +37,19 @@ DEFAULT_ES_SERVER_PORT = 9200 DEFAULT_INDEX = 'freezer' +class MergeMappingException(Exception): + pass + + class ElastichsearchEngine(object): - def __init__(self, es_url, es_index, test_only, always_yes, verbose): + def __init__(self, es_url, es_index, args): self.es_url = es_url self.es_index = es_index - self.test_only = test_only - self.always_yes = always_yes - self.verbose = verbose + self.args = args + self.exit_code = os.EX_OK - def verbose_print(self, message): - if self.verbose: + def verbose_print(self, message, level=1): + if self.args.verbose >= level: print(message) def put_mappings(self, mappings): @@ -56,6 +59,7 @@ class ElastichsearchEngine(object): print '{0}/{1} MATCHES'.format(self.es_index, es_type) else: self.askput_mapping(es_type, mapping) + return self.exit_code def check_index_exists(self): url = '{0}/{1}'.format(self.es_url, self.es_index) @@ -80,16 +84,47 @@ class ElastichsearchEngine(object): return mapping == current_mappings.get(es_type, {}) def askput_mapping(self, es_type, mapping): - if self.test_only: + if self.args.test_only: print '{0}/{1} DOES NOT MATCH'.format(self.es_index, es_type) + self.exit_code = os.EX_DATAERR return - prompt_message = ('{0}/{1}/{2} needs to be deleted. ' + prompt_message = ('{0}/{1}/{2} needs to be updated. ' + 'Proceed ? (y/n)' + .format(self.es_url, + self.es_index, + es_type)) + if not self.proceed(prompt_message, self.args.yes): + return + + self.verbose_print('Trying to upload mappings ...') + try: + self.put_mapping(es_type, mapping) + except MergeMappingException as e: + self.verbose_print('Unable to merge mappings.') + self.verbose_print(e, 2) + else: + print "Mappings updated" + return + + if self.args.yes and not self.args.erase: + # explicit consent to update without explicit consent to erase: + # do not erase type and return error code + self.exit_code = os.EX_DATAERR + print ('{0}/{1} DOES NOT MATCH. ' + 'Need explicit consent to erase types' + .format(self.es_index, es_type)) + return + prompt_message = ('Type {0}/{1}/{2} needs to be deleted. ' 'Proceed (y/n) ? '.format(self.es_url, self.es_index, es_type)) - if self.always_yes or self.proceed(prompt_message): - self.delete_type(es_type) - self.put_mapping(es_type, mapping) + if not self.proceed(prompt_message, self.args.erase): + return + + self.verbose_print('Deleting type {0}'.format(es_type)) + self.delete_type(es_type) + self.verbose_print('Uploading mappings ...') + self.put_mapping(es_type, mapping) def delete_type(self, es_type): url = '{0}/{1}/{2}'.format(self.es_url, self.es_index, es_type) @@ -110,11 +145,11 @@ class ElastichsearchEngine(object): if r.status_code == requests.codes.OK: print "Type {0} mapping created".format(url) else: - raise Exception('Type mapping creation error {0}: ' - '{1}'.format(r.status_code, r.text)) + raise MergeMappingException('Type mapping creation error {0}: ' + '{1}'.format(r.status_code, r.text)) - def proceed(self, message): - if self.always_yes: + def proceed(self, message, assume_yes=False): + if assume_yes: return True while True: selection = raw_input(message) @@ -124,7 +159,7 @@ class ElastichsearchEngine(object): return False -def get_args(): +def get_args(mapping_choices): arg_parser = argparse.ArgumentParser() arg_parser.add_argument( 'host', action='store', default='', nargs='?', @@ -134,16 +169,27 @@ def get_args(): help=('The DB server port ' '(default: {0})'.format(DEFAULT_ES_SERVER_PORT)), dest='port', default=0) + arg_parser.add_argument( + '-m', '--mapping', action='store', + help=('Specific mapping to upload. Valid choices: {0}' + .format(','.join(mapping_choices))), + choices=mapping_choices, + dest='select_mapping', default='') arg_parser.add_argument( '-i', '--index', action='store', help='The DB index (default "{0}")'.format(DEFAULT_INDEX), dest='index') arg_parser.add_argument( '-y', '--yes', action='store_true', - help="Automatic confirmation to index removal", + help="Automatic confirmation to mapping update", dest='yes', default=False) arg_parser.add_argument( - '-v', '--verbose', action='store_true', + '-e', '--erase', action='store_true', + help=("Enable index deletion in case mapping update " + "fails due to incompatible changes"), + dest='erase', default=False) + arg_parser.add_argument( + '-v', '--verbose', action='count', help="Verbose", dest='verbose', default=False) arg_parser.add_argument( @@ -241,29 +287,29 @@ def get_db_params(args): def main(): - args = get_args() + mappings = db_mappings.get_mappings() + + args = get_args(mapping_choices=mappings.keys()) elasticsearch_url, elasticsearch_index = get_db_params(args) es_manager = ElastichsearchEngine(es_url=elasticsearch_url, es_index=elasticsearch_index, - test_only=args.test_only, - always_yes=args.yes, - verbose=args.verbose) - + args=args) if args.verbose: print " db url: {0}".format(elasticsearch_url) print "db index: {0}".format(elasticsearch_index) - mappings = db_mappings.get_mappings() + if args.select_mapping: + mappings = {args.select_mapping: mappings[args.select_mapping]} try: - es_manager.put_mappings(mappings) + exit_code = es_manager.put_mappings(mappings) except Exception as e: print "ERROR {0}".format(e) return os.EX_DATAERR - return os.EX_OK + return exit_code if __name__ == '__main__': sys.exit(main()) diff --git a/freezer_api/common/db_mappings.py b/freezer_api/common/db_mappings.py index 548dbe0b..b069c661 100644 --- a/freezer_api/common/db_mappings.py +++ b/freezer_api/common/db_mappings.py @@ -21,121 +21,161 @@ Hudson (tjh@cryptsoft.com). clients_mapping = { - u'properties': { - u'client': { - u'properties': { - u'client_id': { - u'index': u'not_analyzed', - u'type': u'string', + "properties": { + "client": { + "properties": { + "client_id": { + "index": "not_analyzed", + "type": "string", }, - u'config_id': { - u'index': u'not_analyzed', - u'type': u'string', + "config_id": { + "index": "not_analyzed", + "type": "string", }, - u'description': { - u'type': u'string', + "description": { + "type": "string", }, - u'hostname': { - u'type': u'string', + "hostname": { + "type": "string", }, }, }, - u'user_id': { - u'index': u'not_analyzed', - u'type': u'string', + "user_id": { + "index": "not_analyzed", + "type": "string", + }, + "uuid": { + "index": "not_analyzed", + "type": "string" }, }, } backups_mapping = { - u'properties': { - u'backup_id': { - u'index': u'not_analyzed', - u'type': u'string', + "properties": { + "backup_id": { + "index": "not_analyzed", + "type": "string", }, - u'backup_metadata': { - u'properties': { - u'backup_name': { - u'index': u'not_analyzed', - u'type': u'string', + "backup_metadata": { + "properties": { + "action": { + "type": "string", }, - u'backup_session': { - u'type': u'long', + "always_level": { + "type": "boolean", }, - u'backup_size_compressed': { - u'type': u'long', + "backup_media": { + "type": "string", }, - u'backup_size_uncompressed': { - u'type': u'long', + "backup_name": { + "index": "not_analyzed", + "type": "string", }, - u'broken_links': { - u'index': u'not_analyzed', - u'type': u'string', + "backup_session": { + "type": "long", }, - u'cli': { - u'type': u'string', + "backup_size_compressed": { + "type": "long", }, - u'client_os': { - u'type': u'string', + "backup_size_uncompressed": { + "type": "long", }, - u'compression_alg': { - u'type': u'string', + "broken_links": { + "index": "not_analyzed", + "type": "string", }, - u'container': { - u'index': u'not_analyzed', - u'type': u'string', + "cli": { + "type": "string", }, - u'encrypted': { - u'type': u'boolean', + "client_os": { + "type": "string", }, - u'excluded_files': { - u'type': u'string', + "client_version": { + "type": "string", }, - u'fs_real_path': { - u'type': u'string', + "compression_alg": { + "type": "string", }, - u'host_name': { - u'index': u'not_analyzed', - u'type': u'string', + "container": { + "index": "not_analyzed", + "type": "string", }, - u'level': { - u'type': u'long', + "container_segments": { + "type": "string", }, - u'max_level': { - u'type': u'long', + "curr_backup_level": { + "type": "string", }, - u'mode': { - u'type': u'string', + "current_level": { + "type": "string", }, - u'timestamp': { - u'type': u'long', + "dry_run": { + "type": "boolean", }, - u'total_backup_session_size': { - u'type': u'long', + "encrypted": { + "type": "boolean", }, - u'total_broken_links': { - u'type': u'long', + "excluded_files": { + "type": "string", }, - u'total_directories': { - u'type': u'long', + "fs_real_path": { + "type": "string", }, - u'total_fs_files': { - u'type': u'long', + "host_name": { + "index": "not_analyzed", + "type": "string", }, - u'version': { - u'type': u'string', + "hostname": { + "type": "string", }, - u'vol_snap_path': { - u'type': u'string', + "level": { + "type": "long", + }, + "max_level": { + "type": "long", + }, + "meta_data_file": { + "type": "string", + }, + "mode": { + "type": "string", + }, + "path_to_backup": { + "type": "string", + }, + "time_stamp": { + "type": "string", + }, + "timestamp": { + "type": "long", + }, + "total_backup_session_size": { + "type": "long", + }, + "total_broken_links": { + "type": "long", + }, + "total_directories": { + "type": "long", + }, + "total_fs_files": { + "type": "long", + }, + "version": { + "type": "string", + }, + "vol_snap_path": { + "type": "string", }, }, }, - u'user_id': { - u'index': u'not_analyzed', - u'type': u'string', + "user_id": { + "index": "not_analyzed", + "type": "string", }, - u'user_name': { - u'type': u'string', + "user_name": { + "type": "string", }, }, } @@ -165,6 +205,9 @@ jobs_mapping = { "dry_run": { "type": "boolean" }, + "log_file": { + "type": "string" + }, "lvm_auto_snap": { "type": "string" }, @@ -203,6 +246,9 @@ jobs_mapping = { }, "restore_abs_path": { "type": "string" + }, + "restore_from_host": { + "type": "string" } } }, @@ -279,7 +325,7 @@ jobs_mapping = { def get_mappings(): return { - u'jobs': jobs_mapping, - u'backups': backups_mapping, - u'clients': clients_mapping + "jobs": jobs_mapping, + "backups": backups_mapping, + "clients": clients_mapping } diff --git a/tests/test_db_init.py b/tests/test_db_init.py index 261b7741..4b6bdfb9 100644 --- a/tests/test_db_init.py +++ b/tests/test_db_init.py @@ -31,7 +31,8 @@ from freezer_api.cmd.db_init import (ElastichsearchEngine, parse_config_file, get_db_params, main, - DEFAULT_CONF_PATH) + DEFAULT_CONF_PATH, + MergeMappingException) from freezer_api.common import db_mappings @@ -45,11 +46,15 @@ class TestElasticsearchEngine(unittest.TestCase): } self.mock_resp = Mock() + self.mock_args = Mock() + self.mock_args.test_only = False + self.mock_args.always_yes = False + self.mock_args.verbose = 1 + self.mock_args.select_mapping = '' + self.mock_args.erase = False self.es_manager = ElastichsearchEngine(es_url='http://test:9333', es_index='freezerindex', - test_only=False, - always_yes=False, - verbose=True) + args=self.mock_args) def test_new(self): self.assertIsInstance(self.es_manager, ElastichsearchEngine) @@ -72,17 +77,19 @@ class TestElasticsearchEngine(unittest.TestCase): @patch.object(ElastichsearchEngine, 'proceed') @patch.object(ElastichsearchEngine, 'delete_type') @patch.object(ElastichsearchEngine, 'put_mapping') - def test_askput_calls_delete_and_put_mappunts_when_always_yes(self, + def test_askput_calls_delete_and_put_mappings_when_always_yes_and_erase(self, mock_put_mapping, mock_delete_type, mock_proceed): - self.es_manager.always_yes = True + self.mock_args.yes = True + self.mock_args.erase = True + mock_put_mapping.side_effect = [MergeMappingException('regular test failure'), 0] res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs']) + self.assertTrue(mock_put_mapping.called) mock_delete_type.assert_called_once_with('jobs') - mock_put_mapping.assert_called_once_with('jobs', self.test_mappings['jobs']) def test_askput_does_nothing_when_test_only(self): - self.es_manager.test_only = True + self.mock_args.test_only = True res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs']) self.assertEquals(None, res) @@ -176,8 +183,7 @@ class TestElasticsearchEngine(unittest.TestCase): _raw_input.assert_called_once_with('are you drunk ?') def test_proceed_returns_true_when_always_yes(self): - self.es_manager.always_yes = True - res = self.es_manager.proceed('ask me not') + res = self.es_manager.proceed('ask me not', True) self.assertTrue(res) @patch('freezer_api.cmd.db_init.requests') @@ -210,12 +216,20 @@ class TestElasticsearchEngine(unittest.TestCase): class TestDbInit(unittest.TestCase): + def setUp(self): + self.mock_args = Mock() + self.mock_args.test_only = False + self.mock_args.always_yes = False + self.mock_args.verbose = 1 + self.mock_args.select_mapping = '' + self.mock_args.erase = False + @patch('freezer_api.cmd.db_init.argparse.ArgumentParser') def test_get_args_calls_add_argument(self, mock_ArgumentParser): mock_arg_parser = Mock() mock_ArgumentParser.return_value = mock_arg_parser - retval = get_args() + retval = get_args([]) call_count = mock_arg_parser.add_argument.call_count self.assertGreater(call_count, 6) @@ -262,8 +276,11 @@ class TestDbInit(unittest.TestCase): @patch('freezer_api.cmd.db_init.get_args') def test_main_calls_esmanager_put_mappings_with_mappings(self, mock_get_args, mock_get_db_params, mock_ElastichsearchEngine): + mock_get_args.return_value = self.mock_args mock_get_db_params.return_value = Mock(), Mock() mock_es_manager = Mock() + mock_es_manager.put_mappings.return_value = os.EX_OK + mock_ElastichsearchEngine.return_value = mock_es_manager res = main() @@ -276,6 +293,7 @@ class TestDbInit(unittest.TestCase): @patch('freezer_api.cmd.db_init.get_args') def test_main_return_EX_DATAERR_exitcode_on_error(self, mock_get_args, mock_get_db_params, mock_ElastichsearchEngine): + mock_get_args.return_value = self.mock_args mock_get_db_params.return_value = Mock(), Mock() mock_es_manager = Mock() mock_ElastichsearchEngine.return_value = mock_es_manager