openstack/freezer-api
Code Issues Proposed changes

Adopt for modern elasticsearch library

Browse Source 
U-C were updated to have elasticsearch===9.3.0

Thus, driver needs to be updated to support the modern elasticsearch.

Change-Id: I65007a1fafc05b22431a358c1e88e465b24f3c32
Signed-off-by: Dmitriy Rabotyagov <noonedeadpunk@gmail.com>
(cherry picked from commit c3aed91854)
This commit is contained in:
Dmitriy Rabotyagov
2026-03-12 13:34:43 +01:00
parent 1893240456
commit d21483691f
6 changed files with 107 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
freezer_api/db/elasticsearch/es_manager.py
View File

@@ -61,8 +61,7 @@ class ElasticSearchManager(object):
def _check_mapping_exists(self, mappings):
LOG.info('check if mappings: {0} exists or not'.format(mappings))
return self.elk.indices.exists_type(index=self.index,
doc_type=mappings)
return self.elk.indices.exists(index=self.index)
def get_required_mappings(self):
"""
@@ -149,14 +148,12 @@ class ElasticSearchManager(object):
)
if do_update:
# Call elasticsearch library and put the mappings
return self.elk.indices.put_mapping(doc_type=doc_type,
body=body,
index=self.index
)
return self.elk.indices.put_mapping(body=body,
index=self.index)
else:
return {'acknowledged': False}
return self.elk.indices.put_mapping(doc_type=doc_type, body=body,
index=self.index)
return self.elk.indices.put_mapping(body=body, index=self.index)
def remove_one_mapping(self, doc_type):
"""
@@ -164,13 +161,9 @@ class ElasticSearchManager(object):
:param doc_type: document type to be removed
:return: dict
"""
LOG.info('Removing mapping {0} from index {1}'.format(doc_type,
self.index))
try:
return self.elk.indices.delete_mapping(self.index,
doc_type=doc_type)
except Exception:
raise
LOG.warning('delete_mapping is not supported by the installed '
'elasticsearch client. Mapping %s not removed.',
doc_type)
def remove_mappings(self):
"""

35
freezer_api/storage/elastic.py
View File

@@ -65,7 +65,7 @@ class TypeManager(object):
doc_type=self.doc_type,
id=doc_id)
doc = res['_source']
except elasticsearch.TransportError:
except elasticsearch.NotFoundError:
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('No document found with ID %s') % doc_id)
except Exception as e:
@@ -102,11 +102,10 @@ class TypeManager(object):
created = res['created']
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.ConflictError as e:
raise freezer_api_exc.DocumentExists(message=str(e))
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.StorageEngineError(
message=_i18n._('index operation failed %s') % e)
raise freezer_api_exc.StorageEngineError(message=str(e))
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('index operation failed %s') % e)
@@ -194,13 +193,12 @@ class JobTypeManager(TypeManager):
id=job_id, body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
except elasticsearch.ConflictError as e:
raise freezer_api_exc.DocumentExists(message=str(e))
except elasticsearch.NotFoundError:
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('Unable to find job to update '
'with id %(id)s. %(e)s') % {'id': job_id,
'e': e})
'with id %(id)s.') % {'id': job_id})
except Exception:
raise freezer_api_exc.StorageEngineError(
message=_i18n._('Unable to update job with id %s') % job_id)
@@ -229,9 +227,9 @@ class ActionTypeManager(TypeManager):
id=action_id, body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
except elasticsearch.ConflictError as e:
raise freezer_api_exc.DocumentExists(message=str(e))
except elasticsearch.NotFoundError:
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('Unable to find action to update '
'with id %s') % action_id)
@@ -264,12 +262,13 @@ class SessionTypeManager(TypeManager):
id=session_id, body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
except elasticsearch.ConflictError as e:
raise freezer_api_exc.DocumentExists(message=str(e))
except elasticsearch.NotFoundError:
raise freezer_api_exc.DocumentNotFound(
message=_i18n._('Unable to update session '
'%(id)s %(e)s') % {'id': session_id, 'e': e}
message=_i18n._(
'Unable to update session %(id)s'
) % {'id': session_id}
)
except Exception:

31
freezer_api/storage/elasticv2.py
View File

@@ -77,9 +77,9 @@ class TypeManagerV2(object):
doc_type=self.doc_type,
id=doc_id)
doc = res['_source']
except elasticsearch.TransportError:
except elasticsearch.NotFoundError:
raise freezer_api_exc.DocumentNotFound(
message='No document found with ID:{0}'.format(doc_id))
message='No document found with ID {0}'.format(doc_id))
except Exception as e:
raise freezer_api_exc.StorageEngineError(
message='Get operation failed: {}'.format(e))
@@ -126,9 +126,9 @@ class TypeManagerV2(object):
created = res['created']
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.ConflictError as e:
raise freezer_api_exc.DocumentExists(message=str(e))
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
raise freezer_api_exc.StorageEngineError(
message='index operation failed {0}'.format(e))
except Exception as e:
@@ -240,12 +240,12 @@ class JobTypeManagerV2(TypeManagerV2):
id=job_id, body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
except elasticsearch.ConflictError as e:
raise freezer_api_exc.DocumentExists(message=str(e))
except elasticsearch.NotFoundError:
raise freezer_api_exc.DocumentNotFound(
message='Unable to find job to update with id'
' {0} {1}'.format(job_id, e))
' {0}'.format(job_id))
except Exception:
raise freezer_api_exc.StorageEngineError(
message='Unable to update job with id {0}'.format(job_id))
@@ -280,9 +280,9 @@ class ActionTypeManagerV2(TypeManagerV2):
id=action_id, body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
except elasticsearch.ConflictError as e:
raise freezer_api_exc.DocumentExists(message=str(e))
except elasticsearch.NotFoundError:
raise freezer_api_exc.DocumentNotFound(
message='Unable to find action to update with id'
' {0}'.format(action_id))
@@ -321,12 +321,11 @@ class SessionTypeManagerV2(TypeManagerV2):
id=session_id, body=update_doc)
version = res['_version']
self.es.indices.refresh(index=self.index)
except elasticsearch.TransportError as e:
if e.status_code == 409:
raise freezer_api_exc.DocumentExists(message=e.error)
except elasticsearch.ConflictError as e:
raise freezer_api_exc.DocumentExists(message=str(e))
except elasticsearch.NotFoundError:
raise freezer_api_exc.DocumentNotFound(
message='Unable to update session ID: {0}, '
'Error: {1}'.format(session_id, e))
message='Unable to update session ID: {0}'.format(session_id))
except Exception:
raise freezer_api_exc.StorageEngineError(
message='Unable to update session with id'

54
freezer_api/tests/unit/test_elastic.py
View File

@@ -73,8 +73,10 @@ class TypeManager(common.FreezerBaseTestCase):
self.assertEqual(common.fake_job_0, res)
def test_get_raise_DocumentNotFound_when_doc_not_found(self):
self.mock_es.get.side_effect = elasticsearch.TransportError(
'regular test failure')
meta = mock.Mock()
meta.status = 404
self.mock_es.get.side_effect = elasticsearch.NotFoundError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentNotFound, self.type_manager.get,
user_id=common.fake_job_0_user_id,
doc_id=common.fake_job_0_job_id)
@@ -176,7 +178,7 @@ class TypeManager(common.FreezerBaseTestCase):
def test_insert_raise_StorageEngineError_on_ES_TransportError_exception(
self):
self.mock_es.index.side_effect = elasticsearch.TransportError(
500, 'regular test failure'
'regular test failure'
)
test_doc = {'test_key_412': 'test_value_412', '_version': 5}
self.assertRaises(exceptions.StorageEngineError,
@@ -187,9 +189,10 @@ class TypeManager(common.FreezerBaseTestCase):
def test_insert_raise_DocumentExists_on_ES_TransportError409_exception(
self):
self.mock_es.index.side_effect = elasticsearch.TransportError(
409, 'regular test failure'
)
meta = mock.Mock()
meta.status = 409
self.mock_es.index.side_effect = elasticsearch.ConflictError(
'regular test failure', meta=meta, body={})
test_doc = {'test_key_412': 'test_value_412', '_version': 5}
self.assertRaises(exceptions.DocumentExists, self.type_manager.insert,
doc=test_doc)
@@ -447,16 +450,19 @@ class JobTypeManager(common.FreezerBaseTestCase):
)
def test_update_raise_DocumentNotFound_when_not_found(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
'regular test failure', 1)
meta = mock.Mock()
meta.status = 404
self.mock_es.update.side_effect = elasticsearch.NotFoundError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentNotFound, self.job_manager.update,
job_id=common.fake_job_0_job_id,
job_update_doc={'status': 'sleepy'})
def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
409, 'regular test failure'
)
meta = mock.Mock()
meta.status = 409
self.mock_es.update.side_effect = elasticsearch.ConflictError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentExists, self.job_manager.update,
job_id=common.fake_job_0_job_id,
job_update_doc={'status': 'sleepy'})
@@ -538,17 +544,20 @@ class ActionTypeManager(common.FreezerBaseTestCase):
)
def test_update_raise_DocumentNotFound_when_not_found(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
'regular test failure', 1)
meta = mock.Mock()
meta.status = 404
self.mock_es.update.side_effect = elasticsearch.NotFoundError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentNotFound,
self.action_manager.update,
action_id='asdfsadf',
action_update_doc={'status': 'sleepy'})
def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
409, 'regular test failure'
)
meta = mock.Mock()
meta.status = 409
self.mock_es.update.side_effect = elasticsearch.ConflictError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentExists,
self.action_manager.update,
action_id='pepepepepe2321',
@@ -632,17 +641,20 @@ class SessionTypeManager(common.FreezerBaseTestCase):
body={"doc": {'status': 'sleepy'}})
def test_update_raise_DocumentNotFound_when_not_found(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
'regular test failure', 1)
meta = mock.Mock()
meta.status = 404
self.mock_es.update.side_effect = elasticsearch.NotFoundError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentNotFound,
self.session_manager.update,
session_id='asdfsadf',
session_update_doc={'status': 'sleepy'})
def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
409, 'regular test failure'
)
meta = mock.Mock()
meta.status = 409
self.mock_es.update.side_effect = elasticsearch.ConflictError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentExists,
self.session_manager.update,
session_id='pepepepepe2321',

54
freezer_api/tests/unit/test_elasticv2.py
View File

@@ -82,8 +82,10 @@ class TypeManagerV2(common.FreezerBaseTestCase):
self.assertEqual(common.fake_job_0, res)
def test_get_raise_DocumentNotFound_when_doc_not_found(self):
self.mock_es.get.side_effect = elasticsearch.TransportError(
'regular test failure')
meta = mock.Mock()
meta.status = 404
self.mock_es.get.side_effect = elasticsearch.NotFoundError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentNotFound, self.type_manager.get,
project_id='tecs',
user_id=common.fake_job_0_user_id,
@@ -202,7 +204,7 @@ class TypeManagerV2(common.FreezerBaseTestCase):
def test_insert_raise_StorageEngineError_on_ES_TransportError_exception(
self):
self.mock_es.index.side_effect = elasticsearch.TransportError(
500, 'regular test failure'
'regular test failure'
)
test_doc = {'test_key_412': 'test_value_412', '_version': 5}
self.assertRaises(exceptions.StorageEngineError,
@@ -213,9 +215,10 @@ class TypeManagerV2(common.FreezerBaseTestCase):
def test_insert_raise_DocumentExists_on_ES_TransportError409_exception(
self):
self.mock_es.index.side_effect = elasticsearch.TransportError(
409, 'regular test failure'
)
meta = mock.Mock()
meta.status = 409
self.mock_es.index.side_effect = elasticsearch.ConflictError(
'regular test failure', meta=meta, body={})
test_doc = {'test_key_412': 'test_value_412', '_version': 5}
self.assertRaises(exceptions.DocumentExists, self.type_manager.insert,
doc=test_doc)
@@ -496,16 +499,19 @@ class JobTypeManagerV2(common.FreezerBaseTestCase):
)
def test_update_raise_DocumentNotFound_when_not_found(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
'regular test failure', 1)
meta = mock.Mock()
meta.status = 404
self.mock_es.update.side_effect = elasticsearch.NotFoundError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentNotFound, self.job_manager.update,
job_id=common.fake_job_0_job_id,
job_update_doc={'status': 'sleepy'})
def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
409, 'regular test failure'
)
meta = mock.Mock()
meta.status = 409
self.mock_es.update.side_effect = elasticsearch.ConflictError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentExists, self.job_manager.update,
job_id=common.fake_job_0_job_id,
job_update_doc={'status': 'sleepy'})
@@ -594,17 +600,20 @@ class ActionTypeManagerV2(common.FreezerBaseTestCase):
)
def test_update_raise_DocumentNotFound_when_not_found(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
'regular test failure', 1)
meta = mock.Mock()
meta.status = 404
self.mock_es.update.side_effect = elasticsearch.NotFoundError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentNotFound,
self.action_manager.update,
action_id='asdfsadf',
action_update_doc={'status': 'sleepy'})
def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
409, 'regular test failure'
)
meta = mock.Mock()
meta.status = 409
self.mock_es.update.side_effect = elasticsearch.ConflictError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentExists,
self.action_manager.update,
action_id='pepepepepe2321',
@@ -695,17 +704,20 @@ class SessionTypeManagerV2(common.FreezerBaseTestCase):
body={"doc": {'status': 'sleepy'}})
def test_update_raise_DocumentNotFound_when_not_found(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
'regular test failure', 1)
meta = mock.Mock()
meta.status = 404
self.mock_es.update.side_effect = elasticsearch.NotFoundError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentNotFound,
self.session_manager.update,
session_id='asdfsadf',
session_update_doc={'status': 'sleepy'})
def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self):
self.mock_es.update.side_effect = elasticsearch.TransportError(
409, 'regular test failure'
)
meta = mock.Mock()
meta.status = 409
self.mock_es.update.side_effect = elasticsearch.ConflictError(
'regular test failure', meta=meta, body={})
self.assertRaises(exceptions.DocumentExists,
self.session_manager.update,
session_id='pepepepepe2321',

3
requirements.txt
View File

@@ -1,8 +1,7 @@
# Requirements lower bounds listed here are our best effort to keep them up to
# date but we do not test them so no guarantee of having them all correct. If
# you find any incorrect lower bounds, let us know or propose a fix.
elasticsearch<3.0.0,>=2.0.0 # Apache-2.0
elasticsearch>=8.0.0 # Apache-2.0
falcon>=1.0.0 # Apache-2.0
jsonschema>=3.2.0 # MIT
keystonemiddleware>=4.17.0 # Apache-2.0