diff --git a/freezer_api/db/elasticsearch/es_manager.py b/freezer_api/db/elasticsearch/es_manager.py index a7e472b0..c1fc877f 100644 --- a/freezer_api/db/elasticsearch/es_manager.py +++ b/freezer_api/db/elasticsearch/es_manager.py @@ -61,8 +61,7 @@ class ElasticSearchManager(object): def _check_mapping_exists(self, mappings): LOG.info('check if mappings: {0} exists or not'.format(mappings)) - return self.elk.indices.exists_type(index=self.index, - doc_type=mappings) + return self.elk.indices.exists(index=self.index) def get_required_mappings(self): """ @@ -149,14 +148,12 @@ class ElasticSearchManager(object): ) if do_update: # Call elasticsearch library and put the mappings - return self.elk.indices.put_mapping(doc_type=doc_type, - body=body, - index=self.index - ) + return self.elk.indices.put_mapping(body=body, + index=self.index) else: return {'acknowledged': False} - return self.elk.indices.put_mapping(doc_type=doc_type, body=body, - index=self.index) + + return self.elk.indices.put_mapping(body=body, index=self.index) def remove_one_mapping(self, doc_type): """ @@ -164,13 +161,9 @@ class ElasticSearchManager(object): :param doc_type: document type to be removed :return: dict """ - LOG.info('Removing mapping {0} from index {1}'.format(doc_type, - self.index)) - try: - return self.elk.indices.delete_mapping(self.index, - doc_type=doc_type) - except Exception: - raise + LOG.warning('delete_mapping is not supported by the installed ' + 'elasticsearch client. Mapping %s not removed.', + doc_type) def remove_mappings(self): """ diff --git a/freezer_api/storage/elastic.py b/freezer_api/storage/elastic.py index 47366028..6a6a21f2 100644 --- a/freezer_api/storage/elastic.py +++ b/freezer_api/storage/elastic.py @@ -65,7 +65,7 @@ class TypeManager(object): doc_type=self.doc_type, id=doc_id) doc = res['_source'] - except elasticsearch.TransportError: + except elasticsearch.NotFoundError: raise freezer_api_exc.DocumentNotFound( message=_i18n._('No document found with ID %s') % doc_id) except Exception as e: @@ -102,11 +102,10 @@ class TypeManager(object): created = res['created'] version = res['_version'] self.es.indices.refresh(index=self.index) + except elasticsearch.ConflictError as e: + raise freezer_api_exc.DocumentExists(message=str(e)) except elasticsearch.TransportError as e: - if e.status_code == 409: - raise freezer_api_exc.DocumentExists(message=e.error) - raise freezer_api_exc.StorageEngineError( - message=_i18n._('index operation failed %s') % e) + raise freezer_api_exc.StorageEngineError(message=str(e)) except Exception as e: raise freezer_api_exc.StorageEngineError( message=_i18n._('index operation failed %s') % e) @@ -194,13 +193,12 @@ class JobTypeManager(TypeManager): id=job_id, body=update_doc) version = res['_version'] self.es.indices.refresh(index=self.index) - except elasticsearch.TransportError as e: - if e.status_code == 409: - raise freezer_api_exc.DocumentExists(message=e.error) + except elasticsearch.ConflictError as e: + raise freezer_api_exc.DocumentExists(message=str(e)) + except elasticsearch.NotFoundError: raise freezer_api_exc.DocumentNotFound( message=_i18n._('Unable to find job to update ' - 'with id %(id)s. %(e)s') % {'id': job_id, - 'e': e}) + 'with id %(id)s.') % {'id': job_id}) except Exception: raise freezer_api_exc.StorageEngineError( message=_i18n._('Unable to update job with id %s') % job_id) @@ -229,9 +227,9 @@ class ActionTypeManager(TypeManager): id=action_id, body=update_doc) version = res['_version'] self.es.indices.refresh(index=self.index) - except elasticsearch.TransportError as e: - if e.status_code == 409: - raise freezer_api_exc.DocumentExists(message=e.error) + except elasticsearch.ConflictError as e: + raise freezer_api_exc.DocumentExists(message=str(e)) + except elasticsearch.NotFoundError: raise freezer_api_exc.DocumentNotFound( message=_i18n._('Unable to find action to update ' 'with id %s') % action_id) @@ -264,12 +262,13 @@ class SessionTypeManager(TypeManager): id=session_id, body=update_doc) version = res['_version'] self.es.indices.refresh(index=self.index) - except elasticsearch.TransportError as e: - if e.status_code == 409: - raise freezer_api_exc.DocumentExists(message=e.error) + except elasticsearch.ConflictError as e: + raise freezer_api_exc.DocumentExists(message=str(e)) + except elasticsearch.NotFoundError: raise freezer_api_exc.DocumentNotFound( - message=_i18n._('Unable to update session ' - '%(id)s %(e)s') % {'id': session_id, 'e': e} + message=_i18n._( + 'Unable to update session %(id)s' + ) % {'id': session_id} ) except Exception: diff --git a/freezer_api/storage/elasticv2.py b/freezer_api/storage/elasticv2.py index b30523cc..55fc7ad2 100644 --- a/freezer_api/storage/elasticv2.py +++ b/freezer_api/storage/elasticv2.py @@ -77,9 +77,9 @@ class TypeManagerV2(object): doc_type=self.doc_type, id=doc_id) doc = res['_source'] - except elasticsearch.TransportError: + except elasticsearch.NotFoundError: raise freezer_api_exc.DocumentNotFound( - message='No document found with ID:{0}'.format(doc_id)) + message='No document found with ID {0}'.format(doc_id)) except Exception as e: raise freezer_api_exc.StorageEngineError( message='Get operation failed: {}'.format(e)) @@ -126,9 +126,9 @@ class TypeManagerV2(object): created = res['created'] version = res['_version'] self.es.indices.refresh(index=self.index) + except elasticsearch.ConflictError as e: + raise freezer_api_exc.DocumentExists(message=str(e)) except elasticsearch.TransportError as e: - if e.status_code == 409: - raise freezer_api_exc.DocumentExists(message=e.error) raise freezer_api_exc.StorageEngineError( message='index operation failed {0}'.format(e)) except Exception as e: @@ -240,12 +240,12 @@ class JobTypeManagerV2(TypeManagerV2): id=job_id, body=update_doc) version = res['_version'] self.es.indices.refresh(index=self.index) - except elasticsearch.TransportError as e: - if e.status_code == 409: - raise freezer_api_exc.DocumentExists(message=e.error) + except elasticsearch.ConflictError as e: + raise freezer_api_exc.DocumentExists(message=str(e)) + except elasticsearch.NotFoundError: raise freezer_api_exc.DocumentNotFound( message='Unable to find job to update with id' - ' {0} {1}'.format(job_id, e)) + ' {0}'.format(job_id)) except Exception: raise freezer_api_exc.StorageEngineError( message='Unable to update job with id {0}'.format(job_id)) @@ -280,9 +280,9 @@ class ActionTypeManagerV2(TypeManagerV2): id=action_id, body=update_doc) version = res['_version'] self.es.indices.refresh(index=self.index) - except elasticsearch.TransportError as e: - if e.status_code == 409: - raise freezer_api_exc.DocumentExists(message=e.error) + except elasticsearch.ConflictError as e: + raise freezer_api_exc.DocumentExists(message=str(e)) + except elasticsearch.NotFoundError: raise freezer_api_exc.DocumentNotFound( message='Unable to find action to update with id' ' {0}'.format(action_id)) @@ -321,12 +321,11 @@ class SessionTypeManagerV2(TypeManagerV2): id=session_id, body=update_doc) version = res['_version'] self.es.indices.refresh(index=self.index) - except elasticsearch.TransportError as e: - if e.status_code == 409: - raise freezer_api_exc.DocumentExists(message=e.error) + except elasticsearch.ConflictError as e: + raise freezer_api_exc.DocumentExists(message=str(e)) + except elasticsearch.NotFoundError: raise freezer_api_exc.DocumentNotFound( - message='Unable to update session ID: {0}, ' - 'Error: {1}'.format(session_id, e)) + message='Unable to update session ID: {0}'.format(session_id)) except Exception: raise freezer_api_exc.StorageEngineError( message='Unable to update session with id' diff --git a/freezer_api/tests/unit/test_elastic.py b/freezer_api/tests/unit/test_elastic.py index 334df06d..cbee4522 100644 --- a/freezer_api/tests/unit/test_elastic.py +++ b/freezer_api/tests/unit/test_elastic.py @@ -73,8 +73,10 @@ class TypeManager(common.FreezerBaseTestCase): self.assertEqual(common.fake_job_0, res) def test_get_raise_DocumentNotFound_when_doc_not_found(self): - self.mock_es.get.side_effect = elasticsearch.TransportError( - 'regular test failure') + meta = mock.Mock() + meta.status = 404 + self.mock_es.get.side_effect = elasticsearch.NotFoundError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentNotFound, self.type_manager.get, user_id=common.fake_job_0_user_id, doc_id=common.fake_job_0_job_id) @@ -176,7 +178,7 @@ class TypeManager(common.FreezerBaseTestCase): def test_insert_raise_StorageEngineError_on_ES_TransportError_exception( self): self.mock_es.index.side_effect = elasticsearch.TransportError( - 500, 'regular test failure' + 'regular test failure' ) test_doc = {'test_key_412': 'test_value_412', '_version': 5} self.assertRaises(exceptions.StorageEngineError, @@ -187,9 +189,10 @@ class TypeManager(common.FreezerBaseTestCase): def test_insert_raise_DocumentExists_on_ES_TransportError409_exception( self): - self.mock_es.index.side_effect = elasticsearch.TransportError( - 409, 'regular test failure' - ) + meta = mock.Mock() + meta.status = 409 + self.mock_es.index.side_effect = elasticsearch.ConflictError( + 'regular test failure', meta=meta, body={}) test_doc = {'test_key_412': 'test_value_412', '_version': 5} self.assertRaises(exceptions.DocumentExists, self.type_manager.insert, doc=test_doc) @@ -447,16 +450,19 @@ class JobTypeManager(common.FreezerBaseTestCase): ) def test_update_raise_DocumentNotFound_when_not_found(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 'regular test failure', 1) + meta = mock.Mock() + meta.status = 404 + self.mock_es.update.side_effect = elasticsearch.NotFoundError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentNotFound, self.job_manager.update, job_id=common.fake_job_0_job_id, job_update_doc={'status': 'sleepy'}) def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 409, 'regular test failure' - ) + meta = mock.Mock() + meta.status = 409 + self.mock_es.update.side_effect = elasticsearch.ConflictError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentExists, self.job_manager.update, job_id=common.fake_job_0_job_id, job_update_doc={'status': 'sleepy'}) @@ -538,17 +544,20 @@ class ActionTypeManager(common.FreezerBaseTestCase): ) def test_update_raise_DocumentNotFound_when_not_found(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 'regular test failure', 1) + meta = mock.Mock() + meta.status = 404 + self.mock_es.update.side_effect = elasticsearch.NotFoundError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentNotFound, self.action_manager.update, action_id='asdfsadf', action_update_doc={'status': 'sleepy'}) def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 409, 'regular test failure' - ) + meta = mock.Mock() + meta.status = 409 + self.mock_es.update.side_effect = elasticsearch.ConflictError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentExists, self.action_manager.update, action_id='pepepepepe2321', @@ -632,17 +641,20 @@ class SessionTypeManager(common.FreezerBaseTestCase): body={"doc": {'status': 'sleepy'}}) def test_update_raise_DocumentNotFound_when_not_found(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 'regular test failure', 1) + meta = mock.Mock() + meta.status = 404 + self.mock_es.update.side_effect = elasticsearch.NotFoundError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentNotFound, self.session_manager.update, session_id='asdfsadf', session_update_doc={'status': 'sleepy'}) def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 409, 'regular test failure' - ) + meta = mock.Mock() + meta.status = 409 + self.mock_es.update.side_effect = elasticsearch.ConflictError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentExists, self.session_manager.update, session_id='pepepepepe2321', diff --git a/freezer_api/tests/unit/test_elasticv2.py b/freezer_api/tests/unit/test_elasticv2.py index 9ba491fe..43266c73 100644 --- a/freezer_api/tests/unit/test_elasticv2.py +++ b/freezer_api/tests/unit/test_elasticv2.py @@ -82,8 +82,10 @@ class TypeManagerV2(common.FreezerBaseTestCase): self.assertEqual(common.fake_job_0, res) def test_get_raise_DocumentNotFound_when_doc_not_found(self): - self.mock_es.get.side_effect = elasticsearch.TransportError( - 'regular test failure') + meta = mock.Mock() + meta.status = 404 + self.mock_es.get.side_effect = elasticsearch.NotFoundError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentNotFound, self.type_manager.get, project_id='tecs', user_id=common.fake_job_0_user_id, @@ -202,7 +204,7 @@ class TypeManagerV2(common.FreezerBaseTestCase): def test_insert_raise_StorageEngineError_on_ES_TransportError_exception( self): self.mock_es.index.side_effect = elasticsearch.TransportError( - 500, 'regular test failure' + 'regular test failure' ) test_doc = {'test_key_412': 'test_value_412', '_version': 5} self.assertRaises(exceptions.StorageEngineError, @@ -213,9 +215,10 @@ class TypeManagerV2(common.FreezerBaseTestCase): def test_insert_raise_DocumentExists_on_ES_TransportError409_exception( self): - self.mock_es.index.side_effect = elasticsearch.TransportError( - 409, 'regular test failure' - ) + meta = mock.Mock() + meta.status = 409 + self.mock_es.index.side_effect = elasticsearch.ConflictError( + 'regular test failure', meta=meta, body={}) test_doc = {'test_key_412': 'test_value_412', '_version': 5} self.assertRaises(exceptions.DocumentExists, self.type_manager.insert, doc=test_doc) @@ -496,16 +499,19 @@ class JobTypeManagerV2(common.FreezerBaseTestCase): ) def test_update_raise_DocumentNotFound_when_not_found(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 'regular test failure', 1) + meta = mock.Mock() + meta.status = 404 + self.mock_es.update.side_effect = elasticsearch.NotFoundError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentNotFound, self.job_manager.update, job_id=common.fake_job_0_job_id, job_update_doc={'status': 'sleepy'}) def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 409, 'regular test failure' - ) + meta = mock.Mock() + meta.status = 409 + self.mock_es.update.side_effect = elasticsearch.ConflictError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentExists, self.job_manager.update, job_id=common.fake_job_0_job_id, job_update_doc={'status': 'sleepy'}) @@ -594,17 +600,20 @@ class ActionTypeManagerV2(common.FreezerBaseTestCase): ) def test_update_raise_DocumentNotFound_when_not_found(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 'regular test failure', 1) + meta = mock.Mock() + meta.status = 404 + self.mock_es.update.side_effect = elasticsearch.NotFoundError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentNotFound, self.action_manager.update, action_id='asdfsadf', action_update_doc={'status': 'sleepy'}) def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 409, 'regular test failure' - ) + meta = mock.Mock() + meta.status = 409 + self.mock_es.update.side_effect = elasticsearch.ConflictError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentExists, self.action_manager.update, action_id='pepepepepe2321', @@ -695,17 +704,20 @@ class SessionTypeManagerV2(common.FreezerBaseTestCase): body={"doc": {'status': 'sleepy'}}) def test_update_raise_DocumentNotFound_when_not_found(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 'regular test failure', 1) + meta = mock.Mock() + meta.status = 404 + self.mock_es.update.side_effect = elasticsearch.NotFoundError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentNotFound, self.session_manager.update, session_id='asdfsadf', session_update_doc={'status': 'sleepy'}) def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): - self.mock_es.update.side_effect = elasticsearch.TransportError( - 409, 'regular test failure' - ) + meta = mock.Mock() + meta.status = 409 + self.mock_es.update.side_effect = elasticsearch.ConflictError( + 'regular test failure', meta=meta, body={}) self.assertRaises(exceptions.DocumentExists, self.session_manager.update, session_id='pepepepepe2321', diff --git a/requirements.txt b/requirements.txt index 61124356..66f3b96c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ # Requirements lower bounds listed here are our best effort to keep them up to # date but we do not test them so no guarantee of having them all correct. If # you find any incorrect lower bounds, let us know or propose a fix. - -elasticsearch<3.0.0,>=2.0.0 # Apache-2.0 +elasticsearch>=8.0.0 # Apache-2.0 falcon>=1.0.0 # Apache-2.0 jsonschema>=3.2.0 # MIT keystonemiddleware>=4.17.0 # Apache-2.0