Add feature: support multiple file-list arguments

This update allows users to provide the download list parameter multiple times.
This would helps avoid situations where only one file is modified,
and users can now provide a list of files to be downloaded later
by logscraper and parsed for tags by logsender.

Change-Id: If9adeb5f9be1e7d79ac05076589374a045246668

ansible/roles/logscraper/README.rst

@@ -28,7 +28,7 @@ for example:
         zuul_api_url:
           - https://zuul.opendev.org/api/tenant/openstack
         insecure: false
-        file_list: /etc/logscraper/download-list-TENANT.yaml
+        file_list: ['/etc/logscraper/download-list-TENANT.yaml']
 
 will deploy service with name: `logscraper@openstack.service`.
 It is because on one service we are able to deploy multiple instances
@@ -68,7 +68,8 @@ and second one for getting logs from `sometenant` tenant.
           insecure: True
           download: true
           download_dir: /mnt/logscraper
-          file_list: /etc/logscraper/my-downloadlist.yaml
+          file_list:
+            - /etc/logscraper/my-downloadlist.yaml
     roles:
       - logscraper
 

ansible/roles/logscraper/defaults/main.yml

@@ -24,7 +24,7 @@ container_images:
 #     insecure: true
 #     download: true
 #     download_dir: /mnt/logscraper/sometenant
-#     file_list: ""
+#     file_list: []
 #     job_name:
 #       - test
 #       - test_new

ansible/roles/logscraper/templates/config.j2

@@ -6,7 +6,7 @@ workers: {{ item['logscraper_workers'] | default(1) }}
 max_skipped: {{ item['max_skipped'] | default(1000) }}
 debug: {{ item['debug'] | default(false) }}
 download: {{ item['download'] | default(true) }}
-file_list: {{ item['file_list'] | default(logscraper_dir + '/download-list-' + item['tenant'] + '.yaml') }}
+file_list: {{ item['file_list'] | default([logscraper_dir + '/download-list-' + item['tenant'] + '.yaml']) | list }}
 directory: {{ item['download_dir'] | default('/tmp/logscraper') }}
 wait_time: {{ item['logscraper_wait_time'] | default(120) }}
 insecure: {{ item['insecure'] | default(false) }}

ansible/roles/logscraper/templates/logscraper.sh.j2

@@ -8,7 +8,9 @@
     --uidmap 1000:{{ logscraper_uid }}:1 \
     --name logscraper-{{ item.tenant }} \
     --volume {{ item.logscraper_dir | default(logscraper_dir) }}:{{ logscraper_dir }}:z \
-    --volume {{ item['file_list'] | default(logscraper_dir + '/download-list-' + item['tenant'] + '.yaml') }}:{{ item['file_list'] | default(logscraper_dir + '/download-list-' + item['tenant'] + '.yaml') }}:z \
+    {% for file_list in item['file_list'] | default([logscraper_dir + '/download-list-' + item['tenant'] + '.yaml']) -%}
+    --volume {{ file_list }}:{{ file_list }}:z \
+    {% endfor -%}
     {% if 'logscraper_custom_ca_crt' in item %}
     --volume {{ item['logscraper_custom_ca_crt'] }}:{{ item['logscraper_custom_ca_crt'] }}:z \
     {% endif %}

ansible/roles/logsender/README.rst

@@ -30,7 +30,7 @@ Example Ansible variables that are configuring service:
         es_insecure: true
         es_index: logstash-logscraper
         download_dir: /mnt/logscraper/sometenant
-        file_list: /etc/logsender/download-list-TENANT.yaml
+        file_list: ['/etc/logsender/download-list-TENANT.yaml']
 
 
 That configuration will will deploy service with name: `logsender-openstack.service`.
@@ -76,7 +76,8 @@ and second one for getting logs from `sometenant` tenant.
           es_index: ""
           es_index_prefix: ""
           download_dir: /mnt/logscraper/sometenant
-          file_list: /etc/logscraper/my-downloadlist.yaml
+          file_list:
+            - /etc/logscraper/my-downloadlist.yaml
     roles:
       - logsender
 

ansible/roles/logsender/defaults/main.yml

@@ -20,6 +20,7 @@ container_images:
 #     es_index: ""
 #     es_insecure: false
 #     download_dir: /mnt/logscraper/sometenant
+#     file_list: []
 #     doc_type: "_doc"
 #     logsender_workers: 1
 #     chunk_size: 1500

ansible/roles/logsender/templates/config.j2

@@ -16,6 +16,6 @@ follow: {{ item['follow'] | default(true) }}
 workers: {{ item['logsender_workers'] | default(1) }}
 ignore_es_status: {{ item['ignore_es_status'] | default(false) }}
 directory: {{ item['download_dir'] | default('/tmp/logscraper') }}
-file_list: {{ item['file_list'] | default(logscraper_dir + '/download-list-' + item['tenant'] + '.yaml') }}
+file_list: {{ item['file_list'] | default([logscraper_dir + '/download-list-' + item['tenant'] + '.yaml']) | list }}
 performance_index_prefix: {{ item['performance_index_prefix'] | default('') }}
 subunit_index_prefix: {{ item['subunit_index_prefix'] | default('') }}

ansible/roles/logsender/templates/logsender.sh.j2

@@ -9,7 +9,9 @@
     --name logsender-{{ item.tenant }} \
     --volume {{ item.download_dir }}:{{ item.download_dir }}:z \
     --volume {{ item.logscraper_dir | default(logscraper_dir) }}:{{ logscraper_dir }}:z \
-    --volume {{ item['file_list'] | default(logscraper_dir + '/download-list-' + item['tenant'] + '.yaml') }}:{{ item['file_list'] | default(logscraper_dir + '/download-list-' + item['tenant'] + '.yaml') }}:z \
+    {% for file_list in item['file_list'] | default([logscraper_dir + '/download-list-' + item['tenant'] + '.yaml']) -%}
+    --volume {{ file_list }}:{{ file_list }}:z \
+    {% endfor -%}
     {% if 'logsender_custom_ca_crt' in item %}
     --volume {{ item['logsender_custom_ca_crt'] }}:{{ item['logsender_custom_ca_crt'] }}:z \
     {% endif %}

logscraper/logscraper.conf.sample

@@ -7,7 +7,7 @@ workers: 1
 max_skipped: 1000
 debug: False
 download: True
-file_list: logscraper/download-list.yaml.sample
+file_list: ['logscraper/download-list.yaml.sample']
 directory: /tmp/logscraper
 wait_time: 120
 insecure: False

logscraper/logscraper.py

@@ -133,7 +133,9 @@ def get_arguments():
                                      "CI job logs into gearman.")
     parser.add_argument("--config", help="Logscraper config file",
                         required=True)
-    parser.add_argument("--file-list", help="File list to download")
+    parser.add_argument("--file-list", help="File list to download. Parameter "
+                        "can be set multiple times.", default=[],
+                        action='append')
     parser.add_argument("--zuul-api-url", help="URL(s) for Zuul API. Parameter"
                         " can be set multiple times.", nargs='+', default=[])
     parser.add_argument("--job-name", help="CI job name(s). Parameter can be "
@@ -496,10 +498,31 @@ def save_build_info(directory, build):
         yaml.dump(build, text_file)
 
 
+def merge_dicts(main, additional):
+    merged = main.copy()
+
+    for k, v in additional.items():
+        if k in merged:
+            if isinstance(merged[k], list) and isinstance(v, list):
+                merged[k].extend(v)
+            elif isinstance(merged[k], dict) and isinstance(v, dict):
+                merged[k] = merge_dicts(merged[k], v)
+            else:
+                logging.critical("Trying to merge incompatible "
+                                 "types", type(merged[k]), type(v))
+        else:
+            merged[k] = v
+
+    return merged
+
+
 def load_config(config_path):
     try:
-        with open(config_path) as f:
+        merged_config = {}
-            return yaml.safe_load(f)
+        for yaml_file in config_path:
+            with open(yaml_file) as f:
+                merged_config = merge_dicts(merged_config, yaml.safe_load(f))
+        return merged_config
     except PermissionError:
         logging.critical("Can not open config file %s" % config_path)
     except FileNotFoundError:

logscraper/logsender.conf.sample

@@ -15,6 +15,6 @@ insecure: False
 ca_file:
 follow: True
 workers: 1
-file_list: logscraper/download-list.yaml.sample
+file_list: ['logscraper/download-list.yaml.sample']
 performance_index_prefix: performance-
 subunit_index_prefix: subunit-

logscraper/logsender.py

@@ -41,6 +41,11 @@ from pathlib import Path
 from ruamel.yaml import YAML
 from subunit2sql.read_subunit import ReadSubunit
 
+try:
+    from logscraper import load_config as l_config
+except ImportError:
+    from logscraper.logscraper import load_config as l_config
+
 
 ###############################################################################
 #                                    CLI                                      #
@@ -50,7 +55,9 @@ def get_arguments():
                                      "and push to the Opensearch service")
     parser.add_argument("--config", help="Logscraper config file",
                         required=True)
-    parser.add_argument("--file-list", help="File list to download")
+    parser.add_argument("--file-list", help="File list to download. Parameter "
+                        "can be set multiple times.", default=[],
+                        action='append')
     parser.add_argument("--directory",
                         help="Directory, where the logs will "
                         "be stored.")
@@ -330,13 +337,11 @@ def open_file(path):
 
 
 def get_file_info(config, build_file):
-    yaml = YAML()
+    config_files = l_config(config)
-    with open_file(config) as f:
+    for f in config_files["files"]:
-        config_files = yaml.load(f)
+        file_name = os.path.basename(f["name"])
-        for f in config_files["files"]:
+        if build_file.endswith(file_name):
-            file_name = os.path.basename(f["name"])
+            return f["name"], f.get('tags', []) + [file_name]
-            if build_file.endswith(file_name):
-                return f["name"], f.get('tags', []) + [file_name]
     return os.path.basename(build_file), [os.path.basename(build_file)]
 
 

logscraper/tests/test_logscraper.py

@@ -552,6 +552,61 @@ class TestScraper(base.TestCase):
                                                  10)
         self.assertEqual(2, len(files))
 
+    @mock.patch('yaml.safe_load')
+    @mock.patch('builtins.open', new_callable=mock.mock_open())
+    def test_load_config(self, mock_open, mock_yaml):
+        config_path = ['/tmp/config_1', '/tmp/config_2']
+        config_1 = {'files': [{
+            'name': 'job-output.txt',
+            'tags': ['console', 'console.html']
+        }, {'name': 'logs/undercloud/var/log/extra/logstash.txt',
+            'tags': ['console', 'postpci']}]}
+        config_2 = {'files': [{
+            'name': 'new-job.txt',
+            'tags': ['new-console']
+        }, {'name': 'logs/some-file.log',
+            'tags': ['postpci']}]}
+        final_config = {'files': [{
+            'name': 'job-output.txt',
+            'tags': ['console', 'console.html']
+        }, {
+            'name': 'logs/undercloud/var/log/extra/logstash.txt',
+            'tags': ['console', 'postpci']
+        }, {
+            'name': 'new-job.txt', 'tags': ['new-console']
+        }, {
+            'name': 'logs/some-file.log', 'tags': ['postpci']}]}
+        mock_yaml.side_effect = [config_1, config_2]
+        parsed_config = logscraper.load_config(config_path)
+        self.assertEqual(final_config, parsed_config)
+
+    @mock.patch('yaml.safe_load')
+    @mock.patch('builtins.open', new_callable=mock.mock_open())
+    def test_load_config_different_keys(self, mock_open, mock_yaml):
+        config_path = ['/tmp/config_1', '/tmp/config_2']
+        config_1 = {'files': [{
+            'name': 'job-output.txt',
+            'tags': ['console', 'console.html']
+        }, {'name': 'logs/undercloud/var/log/extra/logstash.txt',
+            'tags': ['console', 'postpci']}]}
+        config_2 = {'files2': [{
+            'name': 'new-job.txt',
+            'tags': ['new-console']
+        }, {'name': 'logs/some-file.log',
+            'tags': ['postpci']}]}
+        final_config = {'files': [{
+            'name': 'job-output.txt',
+            'tags': ['console', 'console.html']
+        }, {'name': 'logs/undercloud/var/log/extra/logstash.txt',
+            'tags': ['console', 'postpci']}
+        ], 'files2': [{
+            'name': 'new-job.txt',
+            'tags': ['new-console']
+        }, {'name': 'logs/some-file.log', 'tags': ['postpci']}]}
+        mock_yaml.side_effect = [config_1, config_2]
+        parsed_config = logscraper.load_config(config_path)
+        self.assertEqual(final_config, parsed_config)
+
 
 class TestConfig(base.TestCase):
     @mock.patch('logscraper.logscraper.load_config')

logscraper/tests/test_logsender.py

@@ -1157,9 +1157,12 @@ class TestSender(base.TestCase):
             got = logsender.get_timestamp(line)
             self.assertEqual(expected, got)
 
-    @mock.patch('ruamel.yaml.YAML.load')
+    @mock.patch('yaml.safe_load')
+    @mock.patch('builtins.open', new_callable=mock.mock_open())
+    @mock.patch('logscraper.logscraper.load_config')
     @mock.patch('logscraper.logsender.open_file')
-    def test_get_file_info(self, mock_open_file, mock_yaml):
+    def test_get_file_info(self, mock_open_file, mock_load_config, mock_open,
+                           mock_yaml):
         config = {'files': [{
             'name': 'job-output.txt',
             'tags': ['console', 'console.html']