Add possibility to provide custom CA certificate

With this commit, if custom_ca_crt variable is set in
tenant_builds for the tenant, the certificate will be mounted
inside the service container, so the --insecure parameter will be not
needed.

Change-Id: If6e28ecf9b5fcf178d1ab2f974cc574ef0866e37

ansible/roles/logscraper/defaults/main.yml

@@ -29,4 +29,5 @@ container_images:
 #     max_skipped: 100
 #     debug: true
 #     logscraper_wait_time: 120
+#     custom_ca_crt: ""
 tenant_builds: []

ansible/roles/logscraper/templates/logscraper.sh.j2

@@ -8,6 +8,9 @@
     --uidmap 1000:{{ logscraper_uid }}:1 \
     --name logscraper-{{ item.tenant }} \
     --volume {{ item.logscraper_dir | default(logscraper_dir) }}:{{ logscraper_dir }}:z \
+    {% if 'custom_ca_crt' in item %}
+    --volume {{ custom_ca_crt }}:{{ custom_ca_crt}}:z \
+    {% endif %}
     {% if 'download_dir' in item %}
     --volume {{ item.download_dir }}:{{ item.download_dir }}:z \
     {% endif %}
@@ -48,4 +51,7 @@
     {% if 'logscraper_wait_time' in item %}
     --wait-time {{ item['logscraper_wait_time'] }} \
     {% endif %}
+    {% if 'custom_ca_crt' in item %}
+    --ca-file {{ custom_ca_crt }} \
+    {% endif %}
     --follow

ansible/roles/logsender/defaults/main.yml

@@ -26,4 +26,5 @@ container_images:
 #     keep: true
 #     ignore_es_status: false
 #     logsender_wait_time: 60
+#     custom_ca_crt: ""
 tenant_builds: []

ansible/roles/logsender/templates/logsender.sh.j2

@@ -9,6 +9,9 @@
     --name logsender-{{ item.tenant }} \
     --volume {{ item.download_dir }}:{{ item.download_dir }}:z \
     --volume {{ item.logscraper_dir | default(logscraper_dir) }}:{{ logscraper_dir }}:z \
+    {% if 'custom_ca_crt' in item %}
+    --volume {{ custom_ca_crt }}:{{ custom_ca_crt}}:z \
+    {% endif %}
     {{ container_images['logsender'] }} \
     /usr/local/bin/logsender \
     --config {{ logscraper_dir }}/config.yaml \
@@ -54,4 +57,7 @@
     {% if 'logsender_wait_time' in item %}
     --wait-time {{ item['logsender_wait_time'] }} \
     {% endif %}
+    {% if 'custom_ca_crt' in item %}
+    --ca-file {{ custom_ca_crt }} \
+    {% endif %}
     --follow

logscraper/logscraper.py

@@ -149,6 +149,7 @@ def get_arguments():
                         "iteration",
                         type=int,
                         default=120)
+    parser.add_argument("--ca-file", help="Provide custom CA certificate")
     args = parser.parse_args()
     return args
 
@@ -666,10 +667,15 @@ def run_scraping(args, zuul_api_url, job_name=None):
 
 
 def run(args):
+    if args.ca_file:
+        validate_ca = args.ca_file
+    else:
+        validate_ca = args.insecure
+
     for zuul_api_url in args.zuul_api_url:
         if args.job_name:
             jobs_in_zuul = filter_available_jobs(zuul_api_url, args.job_name,
-                                                 args.insecure)
+                                                 validate_ca)
             logging.info("Available jobs for %s are %s" % (
                 zuul_api_url, jobs_in_zuul))
             for job_name in jobs_in_zuul:

logscraper/logsender.py

@@ -89,6 +89,7 @@ def get_arguments():
                         "iteration",
                         type=int,
                         default=120)
+    parser.add_argument("--ca-file", help="Provide custom CA certificate")
     args = parser.parse_args()
     return args
 
@@ -444,6 +445,9 @@ def get_es_client(args):
     if args.username and args.password:
         es_creds["http_auth"] = "%s:%s" % (args.username, args.password)
 
+    if args.ca_file:
+        es_creds['ca_certs'] = args.ca_file
+
     es_client = OpenSearch([es_creds], timeout=60)
     logging.info("Connected to Opensearch: %s" % es_client.info())
     return es_client

logscraper/tests/test_logscraper.py

@@ -148,7 +148,7 @@ class FakeArgs(object):
                  checkpoint_file=None, ignore_checkpoint=None,
                  logstash_url=None, workers=None, max_skipped=None,
                  job_name=None, download=None, directory=None,
-                 config=None, wait_time=None):
+                 config=None, wait_time=None, ca_file=None):
 
         self.zuul_api_url = zuul_api_url
         self.gearman_server = gearman_server
@@ -165,6 +165,7 @@ class FakeArgs(object):
         self.directory = directory
         self.config = config
         self.wait_time = wait_time
+        self.ca_file = ca_file
 
 
 class TestScraper(base.TestCase):