openstack/freezer-api
Code Issues Proposed changes

Change default value of elasticsearch replicas

Browse Source 
After executing "freezer-scheduler register" we face problem that
all shards failed in elasticsearch (ES).

If we use number of replicas "0" according to ES documentation
for one node installation then this issue disappeared.

After "first use" of freezer tool cloud engineer can add
additional ES nodes to cluster and fine tune this value
in configuration file.

Change-Id: I4f2a6562358aefb6244f8d0590a9f138db19a925
This commit is contained in:
vnogin
2016-09-09 13:16:47 +03:00
committed by Pierre-Arthur MATHIEU
parent 4c91e0d9ab
commit d963673bdc
6 changed files with 9 additions and 814 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
etc/freezer/freezer-api.conf.sample
View File

@@ -347,6 +347,8 @@
# path to CA certs on disk (string value)
#ca_certs = <None>
# Number of replicas for elk cluster. Default is 2. Use 0 for no replicas
# Number of replicas for elk cluster. Default is 0. Use 0 for no replicas
# In a production environment, this value should be equal to:
# (Number of elasticsearch node in the cluster - 1)
# (integer value)
#number_of_replicas = 2
#number_of_replicas = 0

385
freezer_api/cmd/db_init.py
View File

@@ -1,385 +0,0 @@
#!/usr/bin/env python2
"""
Copyright 2015 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from __future__ import print_function
import argparse
import json
import os
import re
from six.moves import configparser
from six.moves import input
import sys
import requests
from freezer_api.common import db_mappings
DEFAULT_CONF_PATH = '/etc/freezer/freezer-api.conf'
DEFAULT_ES_SERVER_PORT = 9200
DEFAULT_INDEX = 'freezer'
DEFAULT_REPLICAS = 1
class MergeMappingException(Exception):
pass
class NumberOfReplicasException(Exception):
pass
class ElasticSearchEngine(object):
def __init__(self, es_url, es_index, args):
self.es_url = es_url
self.es_index = es_index
self.args = args
self.exit_code = os.EX_OK
def verbose_print(self, message, level=1):
if self.args.verbose >= level:
print(message)
def set_number_of_replicas(self, n):
if self.number_of_replicas_match(n):
print('Number of replicas matches. '
'Current value is {0}'.format(n))
else:
self.askput_number_of_replicas(n)
def number_of_replicas_match(self, n):
url = '{0}/{1}/_settings'.format(self.es_url,
self.es_index)
self.verbose_print('GET {0}\n'.format(url))
r = requests.get(url)
if r.status_code != requests.codes.OK:
raise Exception("ERROR {0}: {1}".format(r.status_code, r.text))
self.verbose_print("response: {0}".format(r))
settings_dict = r.json()
current_n = int(settings_dict[self.es_index]['settings']
['index']['number_of_replicas'])
self.verbose_print("Current replica number: {0}".format(current_n))
return current_n == int(n)
def askput_number_of_replicas(self, n):
if self.args.test_only:
print("Number of replicas don't match")
self.exit_code = os.EX_DATAERR
return
prompt_message = ('Number of replicas needs to be '
'updated to {0}. '
'Proceed ? (y/n)'
.format(n))
if not self.proceed(prompt_message, self.args.yes):
return
url = '{0}/{1}/_settings'.format(self.es_url, self.es_index)
body_dict = {"number_of_replicas": int(n)}
self.verbose_print('PUT {0}\n{1}'.format(url, body_dict))
r = requests.put(url, data=json.dumps(body_dict))
self.verbose_print("response: {0}".format(r))
if r.status_code == requests.codes.OK:
print("Replica number set to {0}".format(self.args.replicas))
else:
raise NumberOfReplicasException('Error setting the replica '
'number, {0}: {1}'
.format(r.status_code, r.text))
def put_mappings(self, mappings):
self.check_index_exists()
for es_type, mapping in mappings.items():
if self.mapping_match(es_type, mapping):
print('{0}/{1} MATCHES'.format(self.es_index, es_type))
else:
self.askput_mapping(es_type, mapping)
return self.exit_code
def check_index_exists(self):
url = '{0}/{1}'.format(self.es_url, self.es_index)
r = requests.post(url)
if r.status_code not in [requests.codes.OK,
requests.codes.BAD_REQUEST]:
raise Exception('Unable to check/create index {0}. '
'ERROR {1}'.format(url, r.status_code))
def mapping_match(self, es_type, mapping):
url = '{0}/{1}/_mapping/{2}'.format(self.es_url,
self.es_index,
es_type)
self.verbose_print("Getting mappings: http GET {0}".format(url))
r = requests.get(url)
self.verbose_print("response: {0}".format(r))
if r.status_code == requests.codes.NOT_FOUND:
return False
if r.status_code != requests.codes.OK:
raise Exception("ERROR {0}: {1}".format(r.status_code, r.text))
current_mappings = r.json().get(self.es_index, {}).get('mappings', {})
return mapping == current_mappings.get(es_type, {})
def askput_mapping(self, es_type, mapping):
if self.args.test_only:
print('{0}/{1} DOES NOT MATCH'.format(self.es_index, es_type))
self.exit_code = os.EX_DATAERR
return
prompt_message = ('{0}/{1}/{2} needs to be updated. '
'Proceed ? (y/n)'
.format(self.es_url,
self.es_index,
es_type))
if not self.proceed(prompt_message, self.args.yes):
return
self.verbose_print('Trying to upload mappings ...')
try:
self.put_mapping(es_type, mapping)
except MergeMappingException as e:
self.verbose_print('Unable to merge mappings.')
self.verbose_print(e, 2)
else:
print("Mappings updated")
return
if self.args.yes and not self.args.erase:
# explicit consent to update without explicit consent to erase:
# do not erase type and return error code
self.exit_code = os.EX_DATAERR
print('{0}/{1} DOES NOT MATCH. '
'Need explicit consent to erase types'
.format(self.es_index, es_type))
return
prompt_message = ('Type {0}/{1}/{2} needs to be deleted. '
'Proceed (y/n) ? '.format(self.es_url,
self.es_index,
es_type))
if not self.proceed(prompt_message, self.args.erase):
return
self.verbose_print('Deleting type {0}'.format(es_type))
self.delete_type(es_type)
self.verbose_print('Uploading mappings ...')
self.put_mapping(es_type, mapping)
def delete_type(self, es_type):
url = '{0}/{1}/{2}'.format(self.es_url, self.es_index, es_type)
self.verbose_print("DELETE {0}".format(url))
r = requests.delete(url)
self.verbose_print("response: {0}".format(r))
if r.status_code not in [requests.codes.OK, requests.codes.NOT_FOUND]:
raise Exception('Type removal error {0}: '
'{1}'.format(r.status_code, r.text))
def put_mapping(self, es_type, mapping):
url = '{0}/{1}/_mapping/{2}'.format(self.es_url,
self.es_index,
es_type)
self.verbose_print('PUT {0}'.format(url))
r = requests.put(url, data=json.dumps(mapping))
self.verbose_print("response: {0}".format(r))
if r.status_code == requests.codes.OK:
print("Type {0} mapping created".format(url))
else:
raise MergeMappingException('Type mapping creation error {0}: '
'{1}'.format(r.status_code, r.text))
def proceed(self, message, assume_yes=False):
if assume_yes:
return True
while True:
selection = input(message)
if selection.upper() == 'Y':
return True
elif selection.upper() == 'N':
return False
def get_args(mapping_choices):
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
'host', action='store', default='', nargs='?',
help='The DB host address[:port], default "localhost"')
arg_parser.add_argument(
'-p', '--port', action='store', type=int,
help=('The DB server port '
'(default: {0})'.format(DEFAULT_ES_SERVER_PORT)),
dest='port', default=0)
arg_parser.add_argument(
'-m', '--mapping', action='store',
help=('Specific mapping to upload. Valid choices: {0}'
.format(','.join(mapping_choices))),
choices=mapping_choices,
dest='select_mapping', default='')
arg_parser.add_argument(
'-i', '--index', action='store',
help='The DB index (default "{0}")'.format(DEFAULT_INDEX),
dest='index')
arg_parser.add_argument(
'-y', '--yes', action='store_true',
help="Automatic confirmation to update mappings and "
"number-of-replicas",
dest='yes', default=False)
arg_parser.add_argument(
'-e', '--erase', action='store_true',
help=("Enable index deletion in case mapping update "
"fails due to incompatible changes"),
dest='erase', default=False)
arg_parser.add_argument(
'-v', '--verbose', action='count',
help="Verbose",
dest='verbose', default=False)
arg_parser.add_argument(
'-t', '--test-only', action='store_true',
help="Test the validity of the mappings, but take no action",
dest='test_only', default=False)
arg_parser.add_argument(
'-c', '--config-file', action='store',
help='Config file with the db information',
dest='config_file', default='')
arg_parser.add_argument(
'-r', '--replicas', action='store',
help='Set the value for the number replicas in the DB index '
'(default {0} when not specified here nor in config file)'
.format(DEFAULT_REPLICAS),
dest='replicas', default=False)
return arg_parser.parse_args()
def find_config_file():
cwd_config = os.path.join(os.getcwd(), 'freezer-api.conf')
for config_file_path in [cwd_config, DEFAULT_CONF_PATH]:
if os.path.isfile(config_file_path):
return config_file_path
def parse_config_file(fname):
"""
Read host URL from config-file
:param fname: config-file path
:return: (host, port, db_index, number_of_replicas)
"""
if not fname:
return None, 0, None, 0
host, port, index, number_of_replicas = None, 0, None, 0
config = configparser.ConfigParser()
config.read(fname)
try:
if config.has_option('storage', 'endpoint'):
endpoint = config.get('storage', 'endpoint')
elif config.has_option('storage', 'hosts'):
endpoint = config.get('storage', 'hosts')
else:
endpoint = ''
match = re.search(r'^http://([^:]+):([\d]+)', endpoint)
if match:
host = match.group(1)
port = int(match.group(2))
except Exception:
pass
try:
index = config.get('storage', 'index')
except Exception:
pass
try:
number_of_replicas = int(config.get('storage', 'number_of_replicas'))
except Exception:
pass
return host, port, index, number_of_replicas
def get_db_params(args):
"""
Extracts the db configuration parameters either from the provided
command line arguments or searching in the default freezer-api config
file /etc/freezer/freezer-api.conf
:param args: argparsed command line arguments
:return: (elasticsearch_url, elastichsearch_index, number_of_replicas)
"""
conf_fname = args.config_file or find_config_file()
if args.verbose:
print("using config file: {0}".format(conf_fname))
conf_host, conf_port, conf_db_index, number_of_replicas = \
parse_config_file(conf_fname)
# host lookup
# 1) host arg (before ':')
# 2) config file provided
# 3) string 'localhost'
host = args.host or conf_host or 'localhost'
host = host.split(':')[0]
# port lookup
# 1) port arg
# 2) host arg (after ':')
# 3) config file provided
# 4) DEFAULT_ES_SERVER_PORT
match_port = None
match = re.search(r':(\d+)$', args.host)
if match:
match_port = match.groups()[0]
port = args.port or match_port or conf_port or DEFAULT_ES_SERVER_PORT
elasticsearch_url = 'http://{0}:{1}'.format(host, port)
# index lookup
# 1) index args
# 2) config file
# 3) string DEFAULT_INDEX
elasticsearch_index = args.index or conf_db_index or DEFAULT_INDEX
return elasticsearch_url, elasticsearch_index, number_of_replicas
def main():
mappings = db_mappings.get_mappings()
args = get_args(mapping_choices=mappings.keys())
elasticsearch_url, elasticsearch_index, elasticsearch_replicas = \
get_db_params(args)
number_of_replicas = int(args.replicas or
elasticsearch_replicas or
DEFAULT_REPLICAS)
es_manager = ElasticSearchEngine(es_url=elasticsearch_url,
es_index=elasticsearch_index,
args=args)
if args.verbose:
print(" db url: {0}".format(elasticsearch_url))
print("db index: {0}".format(elasticsearch_index))
if args.select_mapping:
mappings = {args.select_mapping: mappings[args.select_mapping]}
try:
es_manager.put_mappings(mappings)
es_manager.set_number_of_replicas(number_of_replicas)
except Exception as e:
print("ERROR {0}".format(e))
return os.EX_DATAERR
return es_manager.exit_code
if __name__ == '__main__':
sys.exit(main())

2
freezer_api/cmd/manage.py
View File

@@ -34,7 +34,7 @@ LOG = log.getLogger(__name__)
DEFAULT_ES_SERVER_PORT = 9200
DEFAULT_INDEX = 'freezer'
DEFAULT_REPLICAS = 1
DEFAULT_REPLICAS = 0
def add_db_opts(subparser):

7
freezer_api/storage/driver.py
View File

@@ -57,9 +57,10 @@ def get_elk_opts():
default=None,
help='path to CA certs on disk'),
cfg.IntOpt('number_of_replicas',
default=2,
help='Number of replicas for elk cluster. Default is 2. '
'Use 0 for no replicas')
default=0,
help='Number of replicas for elk cluster. Default is 0. '
'Use 0 for no replicas. This should be set to (number '
'of node in the ES cluter -1).')
]
return storage_opts

422
freezer_api/tests/unit/test_db_init.py
View File

@@ -1,422 +0,0 @@
"""
Copyright 2015 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import json
import unittest
from mock import Mock, patch
import requests
from freezer_api.cmd.db_init import (ElasticSearchEngine,
get_args,
find_config_file,
parse_config_file,
get_db_params,
main,
DEFAULT_CONF_PATH,
MergeMappingException,
NumberOfReplicasException)
from freezer_api.common import db_mappings
class TestElasticSearchEngine(unittest.TestCase):
def setUp(self):
self.test_mappings = {
'jobs': {"properties": {"job_id": {"type": "string"}}},
'backups': {"properties": {"backup_id": {"type": "string"}}},
'clients': {"properties": {"client_id": {"type": "string"}}},
}
self.mock_resp = Mock()
self.mock_args = Mock()
self.mock_args.test_only = False
self.mock_args.always_yes = False
self.mock_args.verbose = 1
self.mock_args.select_mapping = ''
self.mock_args.erase = False
self.mock_args.replicas = 0
self.es_manager = ElasticSearchEngine(es_url='http://test:9333',
es_index='freezerindex',
args=self.mock_args)
def test_new(self):
self.assertIsInstance(self.es_manager, ElasticSearchEngine)
@patch.object(ElasticSearchEngine, 'check_index_exists')
@patch.object(ElasticSearchEngine, 'mapping_match')
@patch.object(ElasticSearchEngine, 'askput_mapping')
@patch.object(ElasticSearchEngine, 'set_number_of_replicas')
def test_put_mappings_does_nothing_when_mappings_match(self,
mock_set_number_of_replicas,
mock_askput_mapping,
mock_mapping_match,
mock_check_index_exists):
self.es_manager.put_mappings(self.test_mappings)
self.assertEquals(mock_askput_mapping.call_count, 0)
@patch.object(ElasticSearchEngine, 'check_index_exists')
@patch.object(ElasticSearchEngine, 'mapping_match')
@patch.object(ElasticSearchEngine, 'askput_mapping')
@patch.object(ElasticSearchEngine, 'set_number_of_replicas')
def test_put_mappings_calls_askput_when_mappings_match_not(self,
mock_set_number_of_replicas,
mock_askput_mapping,
mock_mapping_match,
mock_check_index_exists):
mock_mapping_match.return_value = False
self.es_manager.put_mappings(self.test_mappings)
self.assertEquals(mock_askput_mapping.call_count, 3)
@patch.object(ElasticSearchEngine, 'proceed')
@patch.object(ElasticSearchEngine, 'delete_type')
@patch.object(ElasticSearchEngine, 'put_mapping')
@patch.object(ElasticSearchEngine, 'set_number_of_replicas')
def test_askput_calls_delete_and_put_mappings_when_always_yes_and_erase(self,
mock_set_number_of_replicas,
mock_put_mapping,
mock_delete_type,
mock_proceed):
self.mock_args.yes = True
self.mock_args.erase = True
mock_put_mapping.side_effect = [MergeMappingException('regular test failure'), 0]
res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs'])
self.assertTrue(mock_put_mapping.called)
mock_delete_type.assert_called_once_with('jobs')
def test_askput_does_nothing_when_test_only(self):
self.mock_args.test_only = True
res = self.es_manager.askput_mapping('jobs', self.test_mappings['jobs'])
self.assertEquals(None, res)
@patch('freezer_api.cmd.db_init.requests')
def test_mapping_match_not_found_returns_false(self, mock_requests):
self.mock_resp.status_code = 404
mock_requests.codes.OK = 200
mock_requests.codes.NOT_FOUND = 404
mock_requests.get.return_value = self.mock_resp
res = self.es_manager.mapping_match('jobs', self.test_mappings['jobs'])
self.assertFalse(res)
@patch('freezer_api.cmd.db_init.requests')
def test_mapping_match_raises_Exception_on_response_not_in_200_404(self, mock_requests):
self.mock_resp.status_code = 500
mock_requests.get.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.NOT_FOUND = 404
self.assertRaises(Exception, self.es_manager.mapping_match,
'jobs', self.test_mappings['jobs'])
@patch('freezer_api.cmd.db_init.requests')
def test_mapping_match_return_true_when_mapping_matches(self, mock_requests):
self.mock_resp.status_code = 200
self.mock_resp.json.return_value = {"freezerindex": {"mappings": {"jobs": {"properties": {"job_id": {"type": "string"}}}}}}
mock_requests.get.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.NOT_FOUND = 404
res = self.es_manager.mapping_match('jobs', self.test_mappings['jobs'])
self.assertTrue(res)
@patch('freezer_api.cmd.db_init.requests')
def test_mapping_match_return_false_when_mapping_matches_not(self, mock_requests):
self.mock_resp.status_code = 200
self.mock_resp.text = '{"freezerindex": {"mappings": {"jobs":{"properties": {"job_id": {"type": "balloon"}}}}}}'
mock_requests.get.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.NOT_FOUND = 404
res = self.es_manager.mapping_match('jobs', self.test_mappings['jobs'])
self.assertFalse(res)
@patch('freezer_api.cmd.db_init.requests')
def test_delete_type_returns_none_on_success(self, mock_requests):
self.mock_resp.status_code = 200
mock_requests.codes.OK = 200
mock_requests.codes.NOT_FOUND = 404
mock_requests.delete.return_value = self.mock_resp
res = self.es_manager.delete_type('jobs')
self.assertIsNone(res)
@patch('freezer_api.cmd.db_init.requests')
def test_delete_type_raises_Exception_on_response_code_not_200(self, mock_requests):
self.mock_resp.status_code = requests.codes.BAD_REQUEST
mock_requests.delete.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.BAD_REQUEST = 400
mock_requests.codes.NOT_FOUND = 404
self.assertRaises(Exception, self.es_manager.delete_type, 'jobs')
@patch('freezer_api.cmd.db_init.requests')
def test_put_mapping_returns_none_on_success(self, mock_requests):
self.mock_resp.status_code = 200
mock_requests.put.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.NOT_FOUND = 404
res = self.es_manager.put_mapping('jobs', self.test_mappings['jobs'])
self.assertIsNone(res)
url = 'http://test:9333/freezerindex/_mapping/jobs'
data = '{"properties": {"job_id": {"type": "string"}}}'
mock_requests.put.assert_called_with(url, data=data)
@patch('freezer_api.cmd.db_init.requests')
def test_put_mapping_raises_Exception_on_response_code_not_200(self, mock_requests):
self.mock_resp.status_code = 500
mock_requests.put.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.NOT_FOUND = 404
self.assertRaises(Exception, self.es_manager.put_mapping, 'jobs', self.test_mappings['jobs'])
def test_proceed_returns_true_on_user_y(self):
# This really mocks 'six.moves.input'. Because of the way mock and six
# replace function, we need to mock it at the source.
with patch('freezer_api.cmd.db_init.input', return_value='y') as _raw_input:
res = self.es_manager.proceed('fancy a drink ?')
self.assertTrue(res)
_raw_input.assert_called_once_with('fancy a drink ?')
def test_proceed_returns_false_on_user_n(self):
# This really mocks 'six.moves.input'. Because of the way mock and six
# replace function, we need to mock it at the source.
with patch('freezer_api.cmd.db_init.input', return_value='n') as _raw_input:
res = self.es_manager.proceed('are you drunk ?')
self.assertFalse(res)
_raw_input.assert_called_once_with('are you drunk ?')
def test_proceed_returns_true_when_always_yes(self):
res = self.es_manager.proceed('ask me not', True)
self.assertTrue(res)
@patch('freezer_api.cmd.db_init.requests')
def test_check_index_exists_ok_when_index_exists(self, mock_requests):
self.mock_resp.status_code = 200
mock_requests.post.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.BAD_REQUEST = 400
res = self.es_manager.check_index_exists()
self.assertEquals(res, None)
@patch('freezer_api.cmd.db_init.requests')
def test_check_index_exists_ok_when_index_not_exists(self, mock_requests):
self.mock_resp.status_code = 400
mock_requests.post.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.BAD_REQUEST = 400
res = self.es_manager.check_index_exists()
self.assertEquals(res, None)
@patch('freezer_api.cmd.db_init.requests')
def test_check_index_raises_Exception_when_return_code_not_in_OK_BADREQ(self, mock_requests):
self.mock_resp.status_code = 500
mock_requests.post.return_value = self.mock_resp
mock_requests.codes.OK = 200
mock_requests.codes.BAD_REQUEST = 400
self.assertRaises(Exception, self.es_manager.check_index_exists)
def test_set_number_of_replicas_returns_none_on_match(self):
self.es_manager.number_of_replicas_match = Mock()
self.es_manager.number_of_replicas_match.return_value = True
res = self.es_manager.set_number_of_replicas(5)
self.assertEquals(res, None)
def test_set_number_of_replicas_calls_askput_when_match_not(self):
self.es_manager.number_of_replicas_match = Mock()
self.es_manager.askput_number_of_replicas = Mock()
self.es_manager.number_of_replicas_match.return_value = False
res = self.es_manager.set_number_of_replicas(5)
self.es_manager.askput_number_of_replicas.assert_called_once_with(5)
self.assertEquals(res, None)
@patch('freezer_api.cmd.db_init.requests')
def test_number_of_replicas_match_returns_true_when_match(self, mock_requests):
self.mock_resp.status_code = 200
self.mock_resp.json.return_value = {"freezerindex": {
"settings": {
"index": {
"creation_date": "1447167673951",
"number_of_replicas": "3",
"number_of_shards": "5",
"uuid": "C63kkECBS4KXNPs-KKysPQ",
"version": {
"created": "1040299"
}
}
}}}
mock_requests.get.return_value = self.mock_resp
mock_requests.codes.OK = 200
res = self.es_manager.number_of_replicas_match(3)
self.assertTrue(res)
@patch('freezer_api.cmd.db_init.requests')
def test_number_of_replicas_match_returns_false_when_match_not(self, mock_requests):
self.mock_resp.status_code = 200
self.mock_resp.json.return_value = {"freezerindex": {
"settings": {
"index": {
"creation_date": "1447167673951",
"number_of_replicas": "3",
"number_of_shards": "5",
"uuid": "C63kkECBS4KXNPs-KKysPQ",
"version": {
"created": "1040299"
}
}
}}}
mock_requests.get.return_value = self.mock_resp
mock_requests.codes.OK = 200
res = self.es_manager.number_of_replicas_match(4)
self.assertFalse(res)
def test_askput_number_of_replicas_sets_error_when_test_only(self):
self.mock_args.test_only = True
res = self.es_manager.askput_number_of_replicas(4)
self.assertIsNone(res)
self.assertEquals(self.es_manager.exit_code, os.EX_DATAERR)
def test_askput_number_of_replicas_returns_none_when_no_proceed(self):
self.mock_args.test_only = False
self.es_manager.proceed = Mock()
self.es_manager.proceed.return_value = False
res = self.es_manager.askput_number_of_replicas(4)
self.assertIsNone(res)
self.assertEquals(self.es_manager.exit_code, os.EX_OK)
@patch('freezer_api.cmd.db_init.requests')
def test_askput_number_of_replicas_uses_correct_number(self, mock_requests):
self.mock_args.test_only = False
self.es_manager.proceed = Mock()
self.es_manager.proceed.return_value = True
self.mock_resp.status_code = 200
mock_requests.codes.OK = 200
mock_requests.put.return_value = self.mock_resp
res = self.es_manager.askput_number_of_replicas(4)
self.assertIsNone(res)
mock_requests.put.assert_called_once_with('http://test:9333/freezerindex/_settings',
data=json.dumps({"number_of_replicas": 4}))
self.assertEquals(self.es_manager.exit_code, os.EX_OK)
@patch('freezer_api.cmd.db_init.requests')
def test_askput_number_of_replicas_raises_NumberOfReplicasException_on_request_error(self, mock_requests):
self.mock_args.test_only = False
self.es_manager.proceed = Mock()
self.es_manager.proceed.return_value = True
self.mock_resp.status_code = 500
mock_requests.codes.OK = 200
mock_requests.put.return_value = self.mock_resp
self.assertRaises(NumberOfReplicasException, self.es_manager.askput_number_of_replicas, 4)
class TestDbInit(unittest.TestCase):
def setUp(self):
self.mock_args = Mock()
self.mock_args.test_only = False
self.mock_args.always_yes = False
self.mock_args.verbose = 1
self.mock_args.select_mapping = ''
self.mock_args.erase = False
self.mock_args.replicas = 9
@patch('freezer_api.cmd.db_init.argparse.ArgumentParser')
def test_get_args_calls_add_argument(self, mock_ArgumentParser):
mock_arg_parser = Mock()
mock_ArgumentParser.return_value = mock_arg_parser
retval = get_args([])
call_count = mock_arg_parser.add_argument.call_count
self.assertGreater(call_count, 6)
@patch('freezer_api.cmd.db_init.os.path.isfile')
@patch('freezer_api.cmd.db_init.os.getcwd')
def test_find_config_file_returns_file_in_cwd(self, mock_os_getcwd, mock_os_path_isfile):
mock_os_getcwd.return_value = '/home/woohoo'
mock_os_path_isfile.return_value = True
res = find_config_file()
self.assertEquals('/home/woohoo/freezer-api.conf', res)
@patch('freezer_api.cmd.db_init.os.path.isfile')
@patch('freezer_api.cmd.db_init.os.getcwd')
def test_find_config_file_returns_defaultfile(self, mock_os_getcwd, mock_os_path_isfile):
mock_os_getcwd.return_value = '/home/woohoo'
mock_os_path_isfile.side_effect = [False, True, False]
res = find_config_file()
self.assertEquals(DEFAULT_CONF_PATH, res)
@patch('freezer_api.cmd.db_init.configparser.ConfigParser')
def test_parse_config_file_return_config_file_params(self, mock_ConfigParser):
mock_config = Mock()
mock_ConfigParser.return_value = mock_config
mock_config.get.side_effect = lambda *x: {('storage', 'endpoint'): 'http://iperuranio:1999',
('storage', 'index'): 'ohyes',
('storage', 'number_of_replicas'): '10'}[x]
host, port, index, replicas = parse_config_file('dontcare')
self.assertEquals(host, 'iperuranio')
self.assertEquals(port, 1999)
self.assertEquals(index, 'ohyes')
self.assertEquals(replicas, 10)
def test_parse_config_file_return_False_values_when_no_config_fname(self):
host, port, index, replicas = parse_config_file(None)
self.assertEquals(host, None)
self.assertEquals(port, 0)
self.assertEquals(index, None)
self.assertEquals(replicas, 0)
@patch('freezer_api.cmd.db_init.parse_config_file')
def test_get_db_params_returns_args_parameters(self, mock_parse_config_file):
mock_parse_config_file.return_value = (None, None, None, None)
mock_args = Mock()
mock_args.host = 'pumpkin'
mock_args.port = 12345
mock_args.index = 'ciccio'
elasticsearch_url, elasticsearch_index, elasticsearch_replicas = get_db_params(mock_args)
self.assertEquals(elasticsearch_url, 'http://pumpkin:12345')
self.assertEquals(elasticsearch_index, 'ciccio')
@patch('freezer_api.cmd.db_init.ElasticSearchEngine')
@patch('freezer_api.cmd.db_init.get_db_params')
@patch('freezer_api.cmd.db_init.get_args')
def test_main_calls_esmanager_put_mappings_with_mappings(self,
mock_get_args,
mock_get_db_params,
mock_es_engine):
mock_get_args.return_value = self.mock_args
mock_get_db_params.return_value = 'url', 'index', 0
mock_es_manager = Mock()
mock_es_manager.exit_code = os.EX_OK
mock_es_engine.return_value = mock_es_manager
res = main()
self.assertEquals(res, os.EX_OK)
mappings = db_mappings.get_mappings()
mock_es_manager.put_mappings.assert_called_with(mappings)
@patch('freezer_api.cmd.db_init.ElasticSearchEngine')
@patch('freezer_api.cmd.db_init.get_db_params')
@patch('freezer_api.cmd.db_init.get_args')
def test_main_return_EX_DATAERR_exitcode_on_error(self, mock_get_args,
mock_get_db_params,
mock_es_engine):
mock_get_args.return_value = self.mock_args
mock_get_db_params.return_value = 'url', 'index', 0
mock_es_manager = Mock()
mock_es_engine.return_value = mock_es_manager
mock_es_manager.put_mappings.side_effect = Exception('test error')
res = main()
self.assertEquals(res, os.EX_DATAERR)

1
setup.cfg
View File

@@ -44,7 +44,6 @@ oslo.config.opts =
freezer-api = freezer_api.common.config:list_opts
console_scripts =
freezer-api = freezer_api.cmd.api:main
freezer-db-init = freezer_api.cmd.db_init:main
freezer-manage = freezer_api.cmd.manage:main
tempest.test_plugins =
freezer_api_tempest_tests = freezer_api.tests.freezer_api_tempest_plugin.plugin:FreezerApiTempestPlugin