Merge "elasticsearch init script"

This commit is contained in:
Jenkins 2015-07-15 16:49:17 +00:00 committed by Gerrit Code Review
commit 3cfef3187d
5 changed files with 782 additions and 0 deletions

View File

@ -0,0 +1,258 @@
#!/usr/bin/env python2
"""
Copyright 2015 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import argparse
import os
import json
import re
import requests
import sys
import ConfigParser
from freezer_api.common import db_mappings
DEFAULT_CONF_PATH = '/etc/freezer-api.conf'
class ElastichsearchEngine(object):
def __init__(self, es_url, es_index, test_only, always_yes, verbose):
self.es_url = es_url
self.es_index = es_index
self.test_only = test_only
self.always_yes = always_yes
self.verbose = verbose
def verbose_print(self, message):
if self.verbose:
print(message)
def put_mappings(self, mappings):
for es_type, mapping in mappings.iteritems():
if self.mapping_match(es_type, mapping):
print '{0}/{1} MATCHES'.format(self.es_index, es_type)
else:
self.askput_mapping(es_type, mapping)
def mapping_match(self, es_type, mapping):
url = '{0}/{1}/_mapping/{2}'.format(self.es_url,
self.es_index,
es_type)
self.verbose_print("Getting mappings: http GET {0}".format(url))
r = requests.get(url)
self.verbose_print("response: {0}".format(r))
if r.status_code == 404: # no index found
return False
if r.status_code != 200:
raise Exception("ERROR {0}: {1}".format(r.status_code, r.text))
current_mappings = json.loads(r.text)[str(self.es_index)]['mappings']
return mapping == current_mappings[es_type]
def askput_mapping(self, es_type, mapping):
if self.test_only:
print '{0}/{1} DOES NOT MATCH'.format(self.es_index, es_type)
return
prompt_message = ('{0}/{1}/{2} needs to be deleted. '
'Proceed (y/n) ? '.format(self.es_url,
self.es_index,
es_type))
if self.always_yes or self.proceed(prompt_message):
self.delete_type(es_type)
self.put_mapping(es_type, mapping)
def delete_type(self, es_type):
url = '{0}/{1}/{2}'.format(self.es_url, self.es_index, es_type)
self.verbose_print("DELETE {0}".format(url))
r = requests.delete(url)
self.verbose_print("response: {0}".format(r))
if r.status_code == 200:
print 'Type {0} DELETED'.format(url)
else:
raise Exception('Type removal error {0}: '
'{1}'.format(r.status_code, r.text))
def put_mapping(self, es_type, mapping):
url = '{0}/{1}/_mapping/{2}'.format(self.es_url,
self.es_index,
es_type)
self.verbose_print('PUT {0}'.format(url))
r = requests.put(url, data=json.dumps(mapping))
self.verbose_print("response: {0}".format(r))
if r.status_code == 200:
print "Type {0} mapping created".format(url)
else:
raise Exception('Type mapping creation error {0}: '
'{1}'.format(r.status_code, r.text))
def proceed(self, message):
if self.always_yes:
return True
while True:
selection = raw_input(message)
if selection.upper() == 'Y':
return True
elif selection.upper() == 'N':
return False
def get_args():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
'host', action='store', default='', nargs='?',
help='The DB host address[:port], default "localhost"')
arg_parser.add_argument(
'-p', '--port', action='store', type=int,
help="The DB server port (default 9200)",
dest='port', default=0)
arg_parser.add_argument(
'-i', '--index', action='store',
help='The DB index (default "freezer")',
dest='index')
arg_parser.add_argument(
'-y', '--yes', action='store_true',
help="Automatic confirmation to index removal",
dest='yes', default=False)
arg_parser.add_argument(
'-v', '--verbose', action='store_true',
help="Verbose",
dest='verbose', default=False)
arg_parser.add_argument(
'-t', '--test-only', action='store_true',
help="Test the validity of the mappings, but take no action",
dest='test_only', default=False)
arg_parser.add_argument(
'-c', '--config-file', action='store',
help='Config file with the db information',
dest='config_file', default='')
return arg_parser.parse_args()
def find_config_file():
cwd_config = os.path.join(os.getcwd(), 'freezer-api.conf')
for config_file_path in [cwd_config, DEFAULT_CONF_PATH]:
if os.path.isfile(config_file_path):
return config_file_path
def parse_config_file(fname):
"""
Read host URL from config-file
:param fname: config-file path
:return: (host, port, db_index)
"""
if not fname:
return None, 0, None
host, port, index = None, 0, None
config = ConfigParser.ConfigParser()
config.read(fname)
try:
endpoint = config.get('storage', 'endpoint')
match = re.search(r'^http://([^:]+):([\d]+)$', endpoint)
if match:
host = match.group(1)
port = int(match.group(2))
except:
pass
try:
index = config.get('storage', 'index')
except:
pass
return host, int(port), index
def get_db_params(args):
"""
Extracts the db configuration parameters either from the provided
command line arguments or searching in the default freezer-api config
file /etc/freezer-api.conf
:param args: argparsed command line arguments
:return: (elasticsearch_url, elastichsearch_index)
"""
conf_fname = args.config_file or find_config_file()
if args.verbose:
print "using config file: {0}".format(conf_fname)
conf_host, conf_port, conf_db_index = parse_config_file(conf_fname)
# host lookup
# 1) host arg (before ':')
# 2) config file provided
# 3) string 'localhost'
host = args.host or conf_host or 'localhost'
host = host.split(':')[0]
# port lookup
# 1) port arg
# 2) host arg (after ':')
# 3) config file provided
# 4) 9200
match_port = None
match = re.search(r'(?:)(\d+)$', args.host)
if match:
match_port = match.group()
port = args.port or match_port or conf_port or 9200
elasticsearch_url = 'http://{0}:{1}'.format(host, port)
# index lookup
# 1) index args
# 2) config file
# 3) string 'freezer'
elasticsearch_index = args.index or conf_db_index or 'freezer'
return elasticsearch_url, elasticsearch_index
def main():
args = get_args()
elasticsearch_url, elasticsearch_index = get_db_params(args)
es_manager = ElastichsearchEngine(es_url=elasticsearch_url,
es_index=elasticsearch_index,
test_only=args.test_only,
always_yes=args.yes,
verbose=args.verbose)
if args.verbose:
print " db url: {0}".format(elasticsearch_url)
print "db index: {0}".format(elasticsearch_index)
mappings = db_mappings.get_mappings()
try:
es_manager.put_mappings(mappings)
except Exception as e:
print "ERROR {0}".format(e)
return os.EX_DATAERR
return os.EX_OK
if __name__ == '__main__':
sys.exit(main())

View File

@ -0,0 +1,284 @@
"""
Copyright 2015 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
clients_mapping = {
u'properties': {
u'client': {
u'properties': {
u'client_id': {
u'index': u'not_analyzed',
u'type': u'string',
},
u'config_id': {
u'index': u'not_analyzed',
u'type': u'string',
},
u'description': {
u'type': u'string',
},
u'hostname': {
u'type': u'string',
},
},
},
u'user_id': {
u'index': u'not_analyzed',
u'type': u'string',
},
},
}
backups_mapping = {
u'properties': {
u'backup_id': {
u'type': u'string',
},
u'backup_metadata': {
u'properties': {
u'backup_name': {
u'index': u'not_analyzed',
u'type': u'string',
},
u'backup_session': {
u'type': u'long',
},
u'backup_size_compressed': {
u'type': u'long',
},
u'backup_size_uncompressed': {
u'type': u'long',
},
u'broken_links': {
u'index': u'not_analyzed',
u'type': u'string',
},
u'cli': {
u'type': u'string',
},
u'client_os': {
u'type': u'string',
},
u'compression_alg': {
u'type': u'string',
},
u'container': {
u'index': u'not_analyzed',
u'type': u'string',
},
u'encrypted': {
u'type': u'boolean',
},
u'excluded_files': {
u'type': u'string',
},
u'fs_real_path': {
u'type': u'string',
},
u'host_name': {
u'index': u'not_analyzed',
u'type': u'string',
},
u'level': {
u'type': u'long',
},
u'max_level': {
u'type': u'long',
},
u'mode': {
u'type': u'string',
},
u'timestamp': {
u'type': u'long',
},
u'total_backup_session_size': {
u'type': u'long',
},
u'total_broken_links': {
u'type': u'long',
},
u'total_directories': {
u'type': u'long',
},
u'total_fs_files': {
u'type': u'long',
},
u'version': {
u'type': u'string',
},
u'vol_snap_path': {
u'type': u'string',
},
},
},
u'user_id': {
u'index': u'not_analyzed',
u'type': u'string',
},
u'user_name': {
u'type': u'string',
},
},
}
jobs_mapping = {
"properties": {
"client_id": {
"index": "not_analyzed",
"type": "string"
},
"description": {
"type": "string"
},
"job_actions": {
"properties": {
"freezer_action": {
"properties": {
"action": {
"type": "string"
},
"backup_name": {
"type": "string"
},
"container": {
"type": "string"
},
"dry_run": {
"type": "boolean"
},
"lvm_auto_snap": {
"type": "string"
},
"lvm_dirmount": {
"type": "string"
},
"lvm_snapname": {
"type": "string"
},
"lvm_snapsize": {
"type": "string"
},
"max_level": {
"type": "long"
},
"max_priority": {
"type": "boolean"
},
"max_segment_size": {
"type": "long"
},
"mode": {
"type": "string"
},
"mysql_conf": {
"type": "string"
},
"path_to_backup": {
"type": "string"
},
"remove_older_than": {
"type": "long"
},
"remove_older_then": {
"type": "long"
},
"restore_abs_path": {
"type": "string"
}
}
},
"mandatory": {
"type": "boolean"
},
"max_retries": {
"type": "long"
},
"max_retries_interval": {
"type": "long"
}
}
},
"job_event": {
"type": "string"
},
"job_id": {
"index": "not_analyzed",
"type": "string"
},
"job_schedule": {
"properties": {
"event": {
"type": "string"
},
"result": {
"type": "string"
},
"schedule_day_of_week": {
"type": "string"
},
"schedule_hour": {
"type": "string"
},
"schedule_interval": {
"type": "string"
},
"schedule_minute": {
"type": "string"
},
"schedule_start_date": {
"format": "dateOptionalTime",
"type": "date"
},
"status": {
"type": "string"
},
"time_created": {
"type": "long"
},
"time_ended": {
"type": "long"
},
"time_started": {
"type": "long"
}
}
},
"session_id": {
"type": "string",
"index": "not_analyzed"
},
"session_tag": {
"type": "long"
},
"user_id": {
"index": "not_analyzed",
"type": "string"
}
}
}
def get_mappings():
return {
u'jobs': jobs_mapping,
u'backups': backups_mapping,
u'clients': clients_mapping
}

View File

@ -41,6 +41,7 @@ data_files =
[entry_points]
console_scripts =
freezer-api = freezer_api.cmd.api:main
freezer-db-init = freezer_api.cmd.db_init:main
[pytests]
where=tests

View File

@ -0,0 +1,238 @@
"""
Copyright 2015 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import os
import unittest
from mock import Mock, patch
from freezer_api.cmd.db_init import (ElastichsearchEngine,
get_args,
find_config_file,
parse_config_file,
get_db_params,
main,
DEFAULT_CONF_PATH)
from freezer_api.common import db_mappings
class TestElasticsearchEngine(unittest.TestCase):
def setUp(self):
self.test_mappings = {
'jobs': {"properties": {"job_id": {"type": "string"}}},
'backups': {"properties": {"backup_id": {"type": "string"}}},
'clients': {"properties": {"client_id": {"type": "string"}}},
}
self.mock_resp = Mock()
self.es_manager = ElastichsearchEngine(es_url='http://test:9333',
es_index='freezerindex',
test_only=False,
always_yes=False,
verbose=True)
def test_new(self):
self.assertIsInstance(self.es_manager, ElastichsearchEngine)
@patch.object(ElastichsearchEngine, 'mapping_match')
@patch.object(ElastichsearchEngine, 'askput_mapping')
def test_put_mappings_does_nothing_when_mappings_match(self, mock_askput_mapping, mock_mapping_match):
self.es_manager.put_mappings(self.test_mappings)
self.assertEquals(mock_askput_mapping.call_count, 0)
@patch.object(ElastichsearchEngine, 'mapping_match')
@patch.object(ElastichsearchEngine, 'askput_mapping')
def test_put_mappings_calls_askput_when_mappings_match_not(self, mock_askput_mapping, mock_mapping_match):
mock_mapping_match.return_value = False
self.es_manager.put_mappings(self.test_mappings)
self.assertEquals(mock_askput_mapping.call_count, 3)
@patch.object(ElastichsearchEngine, 'proceed')
@patch.object(ElastichsearchEngine, 'delete_type')
@patch.object(ElastichsearchEngine, 'put_mapping')
def test_askput_calls_delete_and_put_mappunts_when_always_yes(self,
mock_put_mapping,
mock_delete_type,
mock_proceed):
self.es_manager.always_yes = True
res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs'])
mock_delete_type.assert_called_once_with('jobs')
mock_put_mapping.assert_called_once_with('jobs', self.test_mappings['jobs'])
def test_askput_does_nothing_when_test_only(self):
self.es_manager.test_only = True
res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs'])
self.assertEquals(None, res)
@patch('freezer_api.cmd.db_init.requests')
def test_mapping_match_not_found_returns_false(self, mock_requests):
self.mock_resp.status_code = 404
mock_requests.get.return_value = self.mock_resp
res = self.es_manager.mapping_match('jobs', self.test_mappings['jobs'])
self.assertFalse(res)
@patch('freezer_api.cmd.db_init.requests')
def test_mapping_match_raises_Exception_on_response_not_in_200_404(self, mock_requests):
self.mock_resp.status_code = 500
mock_requests.get.return_value = self.mock_resp
self.assertRaises(Exception, self.es_manager.mapping_match,
'jobs', self.test_mappings['jobs'])
@patch('freezer_api.cmd.db_init.requests')
def test_mapping_match_return_true_when_mapping_matches(self, mock_requests):
self.mock_resp.status_code = 200
self.mock_resp.text = '{"freezerindex": {"mappings": {"jobs":{"properties": {"job_id": {"type": "string"}}}}}}'
mock_requests.get.return_value = self.mock_resp
res = self.es_manager.mapping_match('jobs', self.test_mappings['jobs'])
self.assertTrue(res)
@patch('freezer_api.cmd.db_init.requests')
def test_mapping_match_return_false_when_mapping_matches_not(self, mock_requests):
self.mock_resp.status_code = 200
self.mock_resp.text = '{"freezerindex": {"mappings": {"jobs":{"properties": {"job_id": {"type": "balloon"}}}}}}'
mock_requests.get.return_value = self.mock_resp
res = self.es_manager.mapping_match('jobs', self.test_mappings['jobs'])
self.assertFalse(res)
@patch('freezer_api.cmd.db_init.requests')
def test_delete_type_returns_none_on_success(self, mock_requests):
self.mock_resp.status_code = 200
mock_requests.delete.return_value = self.mock_resp
res = self.es_manager.delete_type('jobs')
self.assertIsNone(res)
@patch('freezer_api.cmd.db_init.requests')
def test_delete_type_raises_Exception_on_response_code_not_200(self, mock_requests):
self.mock_resp.status_code = 400
mock_requests.delete.return_value = self.mock_resp
self.assertRaises(Exception, self.es_manager.delete_type, 'jobs')
@patch('freezer_api.cmd.db_init.requests')
def test_put_mapping_returns_none_on_success(self, mock_requests):
self.mock_resp.status_code = 200
mock_requests.put.return_value = self.mock_resp
res = self.es_manager.put_mapping('jobs', self.test_mappings['jobs'])
self.assertIsNone(res)
url = 'http://test:9333/freezerindex/_mapping/jobs'
data = '{"properties": {"job_id": {"type": "string"}}}'
mock_requests.put.assert_called_with(url, data=data)
@patch('freezer_api.cmd.db_init.requests')
def test_put_mapping_raises_Exception_on_response_code_not_200(self, mock_requests):
self.mock_resp.status_code = 500
mock_requests.put.return_value = self.mock_resp
self.assertRaises(Exception, self.es_manager.put_mapping, 'jobs', self.test_mappings['jobs'])
def test_proceed_returns_true_on_user_y(self):
with patch('__builtin__.raw_input', return_value='y') as _raw_input:
res = self.es_manager.proceed('fancy a drink ?')
self.assertTrue(res)
_raw_input.assert_called_once_with('fancy a drink ?')
def test_proceed_returns_false_on_user_n(self):
with patch('__builtin__.raw_input', return_value='n') as _raw_input:
res = self.es_manager.proceed('are you drunk ?')
self.assertFalse(res)
_raw_input.assert_called_once_with('are you drunk ?')
def test_proceed_returns_true_when_always_yes(self):
self.es_manager.always_yes = True
res = self.es_manager.proceed('ask me not')
self.assertTrue(res)
class TestDbInit(unittest.TestCase):
@patch('freezer_api.cmd.db_init.argparse.ArgumentParser')
def test_get_args_calls_add_argument(self, mock_ArgumentParser):
mock_arg_parser = Mock()
mock_ArgumentParser.return_value = mock_arg_parser
retval = get_args()
call_count = mock_arg_parser.add_argument.call_count
self.assertGreater(call_count, 6)
@patch('freezer_api.cmd.db_init.os.path.isfile')
@patch('freezer_api.cmd.db_init.os.getcwd')
def test_find_config_file_returns_file_in_cwd(self, mock_os_getcwd, mock_os_path_isfile):
mock_os_getcwd.return_value = '/home/woohoo'
mock_os_path_isfile.return_value = True
res = find_config_file()
self.assertEquals('/home/woohoo/freezer-api.conf', res)
@patch('freezer_api.cmd.db_init.os.path.isfile')
@patch('freezer_api.cmd.db_init.os.getcwd')
def test_find_config_file_returns_defaultfile(self, mock_os_getcwd, mock_os_path_isfile):
mock_os_getcwd.return_value = '/home/woohoo'
mock_os_path_isfile.side_effect = [False, True, False]
res = find_config_file()
self.assertEquals(DEFAULT_CONF_PATH, res)
@patch('freezer_api.cmd.db_init.ConfigParser.ConfigParser')
def test_parse_config_file_return_config_file_params(self, mock_ConfigParser):
mock_config = Mock()
mock_ConfigParser.return_value = mock_config
mock_config.get.side_effect = lambda *x: {('storage', 'endpoint'): 'http://iperuranio:1999',
('storage', 'index'): 'ohyes'}[x]
host, port, index = parse_config_file('dontcare')
self.assertEquals(host, 'iperuranio')
self.assertEquals(port, 1999)
self.assertEquals(index, 'ohyes')
@patch('freezer_api.cmd.db_init.parse_config_file')
def test_get_db_params_returns_args_parameters(self, mock_parse_config_file):
mock_parse_config_file.return_value = (None, None, None)
mock_args = Mock()
mock_args.host = 'pumpkin'
mock_args.port = 12345
mock_args.index = 'ciccio'
elasticsearch_url, elasticsearch_index = get_db_params(mock_args)
self.assertEquals(elasticsearch_url, 'http://pumpkin:12345')
self.assertEquals(elasticsearch_index, 'ciccio')
@patch('freezer_api.cmd.db_init.ElastichsearchEngine')
@patch('freezer_api.cmd.db_init.get_db_params')
@patch('freezer_api.cmd.db_init.get_args')
def test_main_calls_esmanager_put_mappings_with_mappings(self, mock_get_args, mock_get_db_params,
mock_ElastichsearchEngine):
mock_get_db_params.return_value = Mock(), Mock()
mock_es_manager = Mock()
mock_ElastichsearchEngine.return_value = mock_es_manager
res = main()
self.assertEquals(res, os.EX_OK)
mappings = db_mappings.get_mappings()
mock_es_manager.put_mappings.assert_called_with(mappings)
@patch('freezer_api.cmd.db_init.ElastichsearchEngine')
@patch('freezer_api.cmd.db_init.get_db_params')
@patch('freezer_api.cmd.db_init.get_args')
def test_main_return_EX_DATAERR_exitcode_on_error(self, mock_get_args, mock_get_db_params,
mock_ElastichsearchEngine):
mock_get_db_params.return_value = Mock(), Mock()
mock_es_manager = Mock()
mock_ElastichsearchEngine.return_value = mock_es_manager
mock_es_manager.put_mappings.side_effect = Exception('test error')
res = main()
self.assertEquals(res, os.EX_DATAERR)

View File

@ -15,6 +15,7 @@ deps =
keystonemiddleware
elasticsearch
jsonschema
mock
install_command = pip install -U {opts} {packages}
setenv = VIRTUAL_ENV={envdir}