From 9f5fe86cb8ec6829fc650785053f600169acc2d9 Mon Sep 17 00:00:00 2001 From: Daniel Pawlik Date: Fri, 3 Jun 2022 14:03:49 +0200 Subject: [PATCH] Add possibility to provide custom CA certificate With this commit, if custom_ca_crt variable is set in tenant_builds for the tenant, the certificate will be mounted inside the service container, so the --insecure parameter will be not needed. Change-Id: If6e28ecf9b5fcf178d1ab2f974cc574ef0866e37 --- ansible/roles/logscraper/defaults/main.yml | 1 + ansible/roles/logscraper/templates/logscraper.sh.j2 | 6 ++++++ ansible/roles/logsender/defaults/main.yml | 1 + ansible/roles/logsender/templates/logsender.sh.j2 | 6 ++++++ logscraper/logscraper.py | 8 +++++++- logscraper/logsender.py | 4 ++++ logscraper/tests/test_logscraper.py | 3 ++- 7 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ansible/roles/logscraper/defaults/main.yml b/ansible/roles/logscraper/defaults/main.yml index 32a6a16..35ed94d 100644 --- a/ansible/roles/logscraper/defaults/main.yml +++ b/ansible/roles/logscraper/defaults/main.yml @@ -29,4 +29,5 @@ container_images: # max_skipped: 100 # debug: true # logscraper_wait_time: 120 +# custom_ca_crt: "" tenant_builds: [] diff --git a/ansible/roles/logscraper/templates/logscraper.sh.j2 b/ansible/roles/logscraper/templates/logscraper.sh.j2 index e3dbb62..e105a99 100644 --- a/ansible/roles/logscraper/templates/logscraper.sh.j2 +++ b/ansible/roles/logscraper/templates/logscraper.sh.j2 @@ -8,6 +8,9 @@ --uidmap 1000:{{ logscraper_uid }}:1 \ --name logscraper-{{ item.tenant }} \ --volume {{ item.logscraper_dir | default(logscraper_dir) }}:{{ logscraper_dir }}:z \ + {% if 'custom_ca_crt' in item %} + --volume {{ custom_ca_crt }}:{{ custom_ca_crt}}:z \ + {% endif %} {% if 'download_dir' in item %} --volume {{ item.download_dir }}:{{ item.download_dir }}:z \ {% endif %} @@ -48,4 +51,7 @@ {% if 'logscraper_wait_time' in item %} --wait-time {{ item['logscraper_wait_time'] }} \ {% endif %} + {% if 'custom_ca_crt' in item %} + --ca-file {{ custom_ca_crt }} \ + {% endif %} --follow diff --git a/ansible/roles/logsender/defaults/main.yml b/ansible/roles/logsender/defaults/main.yml index bd146e2..730e86a 100644 --- a/ansible/roles/logsender/defaults/main.yml +++ b/ansible/roles/logsender/defaults/main.yml @@ -26,4 +26,5 @@ container_images: # keep: true # ignore_es_status: false # logsender_wait_time: 60 +# custom_ca_crt: "" tenant_builds: [] diff --git a/ansible/roles/logsender/templates/logsender.sh.j2 b/ansible/roles/logsender/templates/logsender.sh.j2 index 0c447c7..fc417fd 100644 --- a/ansible/roles/logsender/templates/logsender.sh.j2 +++ b/ansible/roles/logsender/templates/logsender.sh.j2 @@ -9,6 +9,9 @@ --name logsender-{{ item.tenant }} \ --volume {{ item.download_dir }}:{{ item.download_dir }}:z \ --volume {{ item.logscraper_dir | default(logscraper_dir) }}:{{ logscraper_dir }}:z \ + {% if 'custom_ca_crt' in item %} + --volume {{ custom_ca_crt }}:{{ custom_ca_crt}}:z \ + {% endif %} {{ container_images['logsender'] }} \ /usr/local/bin/logsender \ --config {{ logscraper_dir }}/config.yaml \ @@ -54,4 +57,7 @@ {% if 'logsender_wait_time' in item %} --wait-time {{ item['logsender_wait_time'] }} \ {% endif %} + {% if 'custom_ca_crt' in item %} + --ca-file {{ custom_ca_crt }} \ + {% endif %} --follow diff --git a/logscraper/logscraper.py b/logscraper/logscraper.py index 1f1a5e9..4a517ff 100755 --- a/logscraper/logscraper.py +++ b/logscraper/logscraper.py @@ -149,6 +149,7 @@ def get_arguments(): "iteration", type=int, default=120) + parser.add_argument("--ca-file", help="Provide custom CA certificate") args = parser.parse_args() return args @@ -666,10 +667,15 @@ def run_scraping(args, zuul_api_url, job_name=None): def run(args): + if args.ca_file: + validate_ca = args.ca_file + else: + validate_ca = args.insecure + for zuul_api_url in args.zuul_api_url: if args.job_name: jobs_in_zuul = filter_available_jobs(zuul_api_url, args.job_name, - args.insecure) + validate_ca) logging.info("Available jobs for %s are %s" % ( zuul_api_url, jobs_in_zuul)) for job_name in jobs_in_zuul: diff --git a/logscraper/logsender.py b/logscraper/logsender.py index 84c8e6f..80e37fd 100755 --- a/logscraper/logsender.py +++ b/logscraper/logsender.py @@ -89,6 +89,7 @@ def get_arguments(): "iteration", type=int, default=120) + parser.add_argument("--ca-file", help="Provide custom CA certificate") args = parser.parse_args() return args @@ -444,6 +445,9 @@ def get_es_client(args): if args.username and args.password: es_creds["http_auth"] = "%s:%s" % (args.username, args.password) + if args.ca_file: + es_creds['ca_certs'] = args.ca_file + es_client = OpenSearch([es_creds], timeout=60) logging.info("Connected to Opensearch: %s" % es_client.info()) return es_client diff --git a/logscraper/tests/test_logscraper.py b/logscraper/tests/test_logscraper.py index 1a10ce8..3ea7f62 100644 --- a/logscraper/tests/test_logscraper.py +++ b/logscraper/tests/test_logscraper.py @@ -148,7 +148,7 @@ class FakeArgs(object): checkpoint_file=None, ignore_checkpoint=None, logstash_url=None, workers=None, max_skipped=None, job_name=None, download=None, directory=None, - config=None, wait_time=None): + config=None, wait_time=None, ca_file=None): self.zuul_api_url = zuul_api_url self.gearman_server = gearman_server @@ -165,6 +165,7 @@ class FakeArgs(object): self.directory = directory self.config = config self.wait_time = wait_time + self.ca_file = ca_file class TestScraper(base.TestCase):