Remove build dirs with missing files; add more debug logs

It is very rary, but still from time to time there are some build
directories, that does not contain necessary files and can not be
proceded by the logsender.
This commit will provide a file with a timestamp, when the build
directory can not be processed by logsender and if after 12 hours nothing
changed, dir would be removed.
Also added more debug logs to see, what files are empty and skipped or
if buildinfo file is created properly.

Change-Id: Ie2fbde3bf7946a3aa84bfd60f87616afa0510033

logscraper/logscraper.py

@@ -491,6 +491,7 @@ def get_last_job_results(zuul_url, insecure, max_builds, build_cache,
 #                              Log scraper                                    #
 ###############################################################################
 def save_build_info(directory, build):
+    logging.debug("Saving buildinfo in: %s" % directory)
     with open("%s/buildinfo" % directory, "w") as text_file:
         yaml.dump(build, text_file)
 

logscraper/logsender.py

@@ -37,6 +37,7 @@ from ast import literal_eval
 from opensearchpy import exceptions as opensearch_exceptions
 from opensearchpy import helpers
 from opensearchpy import OpenSearch
+from pathlib import Path
 from ruamel.yaml import YAML
 from subunit2sql.read_subunit import ReadSubunit
 
@@ -136,6 +137,21 @@ def read_yaml_file(file_path):
         return yaml.load(f)
 
 
+def remove_old_dir(root, build_uuid, files):
+    # Skip main download directory
+    if not files:
+        return
+
+    min_age = datetime.datetime.utcnow() - datetime.timedelta(hours=12)
+
+    build_dir_path = "%s/%s" % (root, build_uuid)
+    build_age = (Path(root) / build_uuid).stat().st_mtime
+    if min_age.timestamp() > build_age:
+        logging.warning("Some files are still missing for %s and nothing "
+                        "changed for 12 hours." % build_uuid)
+        remove_directory(build_dir_path)
+
+
 def get_inventory_info(directory):
     try:
         return read_yaml_file("%s/inventory.yaml" % directory)
@@ -165,7 +181,7 @@ def get_ready_directories(directory):
         else:
             logging.info("Skipping build with uuid %s. Probably all files "
                          "are not downloaded yet." % build_uuid)
-            continue
+            remove_old_dir(root, build_uuid, files)
 
     return log_files
 
@@ -475,6 +491,8 @@ def send(ready_directory, args, directory, index, perf_index, subunit_index):
         # in the build dir are fine, and the dir is keeped because of it.
         # We don't want to skip removing dir, when one of the file was empty.
         if not _is_file_not_empty("%s/%s" % (build_dir, build_file)):
+            logging.debug("File %s/%s is empty. Skipping..." % (
+                build_dir, build_file))
             continue
 
         fields = copy.deepcopy(es_fields)

logscraper/tests/test_logsender.py

@@ -22,6 +22,7 @@ import os
 from logscraper import logsender
 from logscraper.tests import base
 from opensearchpy.exceptions import TransportError
+from pathlib import Path
 from ruamel.yaml import YAML
 from unittest import mock
 
@@ -915,6 +916,21 @@ class TestSender(base.TestCase):
         self.assertEqual(es_doc, list(mock_bulk.call_args.args[1]))
         self.assertEqual(1, mock_bulk.call_count)
 
+    @mock.patch('logscraper.logsender.remove_directory')
+    @mock.patch.object(Path, 'stat')
+    def test_remove_old_dir(self, mock_stat, mock_rm):
+        mock_stat.return_value.st_mtime = 1685575754.860575
+        logsender.remove_old_dir("Somedir", "someBuildUuid", ["someFile"])
+        self.assertEqual(1, mock_rm.call_count)
+
+    @mock.patch('logscraper.logsender.remove_directory')
+    @mock.patch.object(Path, 'stat')
+    def test_remove_old_dir_keep(self, mock_stat, mock_rm):
+        now = datetime.datetime.utcnow().timestamp()
+        mock_stat.return_value.st_mtime = now
+        logsender.remove_old_dir("Somedir", "someBuildUuid", ["someFile"])
+        self.assertEqual(0, mock_rm.call_count)
+
     @mock.patch('logscraper.logsender.logline_iter')
     def test_doc_iter(self, mock_logline):
         text = [(datetime.datetime(2022, 2, 28, 9, 39, 9, 596000),