From de437439ad602a3b96fa23ef59f703587b87440b Mon Sep 17 00:00:00 2001 From: Ramy Asselin Date: Tue, 18 Oct 2016 05:13:45 -0700 Subject: [PATCH] Wait until the most recent index is available When elastic search indexing is behind, and the day has progressed forward to a new day, the latest index is not yet available for use. Exclude it from searches until it is ready in order to avoid the ElasticHttpNotFoundError. Add Unit tests for this case as well as for when multiple days are specified for the search. Change-Id: Ifd27d1ab21bebcb63b48ea164f425c4a2ac8759c --- elastic_recheck/results.py | 11 ++++-- elastic_recheck/tests/unit/test_results.py | 39 +++++++++++++++++++--- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/elastic_recheck/results.py b/elastic_recheck/results.py index f52d7e45..8eb3dc6e 100644 --- a/elastic_recheck/results.py +++ b/elastic_recheck/results.py @@ -71,11 +71,16 @@ class SearchEngine(object): # today's index datefmt = self._indexfmt now = datetime.datetime.utcnow() - indexes = [now.strftime(datefmt)] + indexes = [] + latest_index = now.strftime(datefmt) + if self._is_valid_index(es, latest_index): + indexes.append(latest_index) if recent: lasthr = now - datetime.timedelta(hours=1) - if lasthr.strftime(datefmt) != now.strftime(datefmt): - indexes.append(lasthr.strftime(datefmt)) + lasthr_index = lasthr.strftime(datefmt) + if lasthr_index != latest_index: + if self._is_valid_index(es, lasthr_index): + indexes.append(lasthr.strftime(datefmt)) for day in range(1, days): lastday = now - datetime.timedelta(days=day) index_name = lastday.strftime(datefmt) diff --git a/elastic_recheck/tests/unit/test_results.py b/elastic_recheck/tests/unit/test_results.py index dc7b0a22..5c271ea3 100644 --- a/elastic_recheck/tests/unit/test_results.py +++ b/elastic_recheck/tests/unit/test_results.py @@ -145,13 +145,42 @@ class TestSearchEngine(tests.TestCase): # The search index comparison goes back one hour and cuts off by day, # so test that we're one hour and one second into today so we only have # one index in the search call. - self._test_search_recent(search_mock, MockDatetimeToday, - expected_indexes=['logstash-2014.06.12']) + with mock.patch.object( + pyelasticsearch.ElasticSearch, 'status') as mock_data: + mock_data.return_value = "Not an exception" + self._test_search_recent(search_mock, MockDatetimeToday, + expected_indexes=['logstash-2014.06.12']) def test_search_recent_multiple_indexes(self, search_mock): # The search index comparison goes back one hour and cuts off by day, # so test that we're 59 minutes and 59 seconds into today so that we # have an index for today and yesterday in the search call. - self._test_search_recent(search_mock, MockDatetimeYesterday, - expected_indexes=['logstash-2014.06.12', - 'logstash-2014.06.11']) + with mock.patch.object( + pyelasticsearch.ElasticSearch, 'status') as mock_data: + mock_data.return_value = "Not an exception" + self._test_search_recent(search_mock, MockDatetimeYesterday, + expected_indexes=['logstash-2014.06.12', + 'logstash-2014.06.11']) + + def test_search_no_indexes(self, search_mock): + # Test when no indexes are valid + with mock.patch.object( + pyelasticsearch.ElasticSearch, 'status') as mock_data: + mock_data.side_effect = pyelasticsearch.exceptions.\ + ElasticHttpNotFoundError() + self._test_search_recent(search_mock, MockDatetimeYesterday, + expected_indexes=[]) + + def test_search_days(self, search_mock): + # Test when specific days are used. + with mock.patch.object( + pyelasticsearch.ElasticSearch, 'status') as mock_data: + mock_data.return_value = "Not an exception" + datetime.datetime = MockDatetimeYesterday + result_set = self.engine.search(self.query, size=10, days=3, + recent=False) + self.assertEqual(0, len(result_set)) + search_mock.assert_called_once_with(self.query, size=10, + index=['logstash-2014.06.12', + 'logstash-2014.06.11', + 'logstash-2014.06.10'])