Merge "Change deploy-service playbook; improve logsender code"

ansible/playbooks/deploy-services.yml

@@ -16,16 +16,14 @@
         es_host: opensearch.logs.openstack.org
         es_port: 443
         es_insecure: false
-        es_index_prefix: 'logstash-logs'
+        es_index_prefix: 'logstash-logs-'
         max_skipped: 1000
         logscraper_workers: 8
         logsender_workers: 2
+  roles:
+    - logscraper
+    - logsender
   tasks:
-    - name: Run check services
-      include_role:
-        name: check-services
-        tasks_from: download.yml
-
     - name: Install firewalld package
       yum:
         name: firewalld

ansible/roles/logscraper/templates/logscraper.sh.j2

@@ -21,26 +21,26 @@
     {% endfor %}
     {% endif %}
     {% if 'gearman_port' in item and 'gearman_server' in item %}
-    --gearman-port {{ item.gearman_port }} \
-    --gearman-server {{ item.gearman_server }} \
+    --gearman-port {{ item['gearman_port'] }} \
+    --gearman-server {{ item['gearman_server'] }} \
     {% else %}
     --download \
     {% endif %}
     {% if 'download_dir' in item %}
-    --directory {{ item.download_dir }} \
+    --directory {{ item['download_dir'] }} \
     {% endif %}
     {% if 'insecure' in item %}
     --insecure \
     {% endif %}
     --checkpoint-file {{ item.checkpoint_file | default(logscraper_dir + '/checkpoint') }} \
     {% if 'logstash_url' in item %}
-    --logstash-url {{ logstash_url }} \
+    --logstash-url {{ item['logstash_url'] }} \
     {% endif %}
     {% if 'logscraper_workers' in item %}
-    --workers {{ logscraper_workers }} \
+    --workers {{ item['logscraper_workers'] }} \
     {% endif %}
     {% if 'max_skipped' in item %}
-    --max-skipped {{ max_skipped }} \
+    --max-skipped {{ item.max_skipped }} \
     {% endif %}
     {% if 'debug' in item %}
     --debug \

logscraper/logsender.py

@@ -201,7 +201,7 @@ def get_timestamp(line):
 
 def get_message(line):
     try:
-        return line.split("|", 1)[1].replace('\n', '')
+        return line.split("|", 1)[1].replace('\n', '').lstrip()
     except IndexError:
         return line.replace('\n', '')
 
@@ -280,10 +280,11 @@ def send(ready_directory, args, directory, index, workers):
     logging.debug("Provided build info %s" % es_fields)
 
     for build_file in build_files:
-        es_fields["filename"] = build_file
-        es_fields["log_url"] = es_fields["log_url"] + build_file
+        fields = copy.deepcopy(es_fields)
+        fields["filename"] = build_file
+        fields["log_url"] = fields["log_url"] + build_file
         send_status = send_to_es("%s/%s" % (build_dir, build_file),
-                                 es_fields, es_client, index, workers,
+                                 fields, es_client, index, workers,
                                  args.chunk_size, args.doc_type)
 
     if args.keep:

logscraper/tests/test_logsender.py

@@ -542,7 +542,7 @@ class TestSender(base.TestCase):
             '_source': {
                 '@timestamp': '2022-02-28T09:39:09.596000',
                 'field': 'test',
-                'message': ' Job console starting...'
+                'message': 'Job console starting...'
             },
             '_type': '_doc'
         }, {
@@ -550,7 +550,7 @@ class TestSender(base.TestCase):
             '_source': {
                 '@timestamp': '2022-02-28T09:39:09.610000',
                 'field': 'test',
-                'message': ' Updating repositories'
+                'message': 'Updating repositories'
             },
             '_type': '_doc'
         }]
@@ -591,8 +591,8 @@ class TestSender(base.TestCase):
     def test_get_message(self):
         line_1 = "28-02-2022 09:44:58.839036 | Some message"
         line_2 = "2022-02-28 09:44:58.839036 | Other message | other log info"
-        self.assertEqual(" Some message", logsender.get_message(line_1))
-        self.assertEqual(" Other message | other log info",
+        self.assertEqual("Some message", logsender.get_message(line_1))
+        self.assertEqual("Other message | other log info",
                          logsender.get_message(line_2))
 
     def test_get_timestamp(self):