Move options into config file

Change-Id: I3aa7ec13a3ee99ece0d3e0a73c83f477a87b8c1b
2022-06-11 11:00:04 +02:00 · 2022-06-11 11:00:04 +02:00 · fe3ee26348
commit fe3ee26348
parent 1990b7c08f
19 changed files with 440 additions and 421 deletions
--- a/README.rst
+++ b/README.rst
@ -216,7 +216,7 @@ To download logs:
    --checkpoint-file /tmp/results-checkpoint \
    --worker 8 \
    --max-skipped 100 \
-    --download \
+    --download True \
    --directory /tmp/logscraper

 This operation took: 30 seconds and it uses 130 MB of RAM.
--- a/ansible/playbooks/check-services.yml
+++ b/ansible/playbooks/check-services.yml
@ -22,6 +22,7 @@
          - https://zuul.opendev.org/api/tenant/openstack
        insecure: false
        job_names: []
+        download: false
  pre_tasks:
    - name: Update all packages
      become: true
--- a/ansible/roles/logscraper/defaults/main.yml
+++ b/ansible/roles/logscraper/defaults/main.yml
@ -22,6 +22,7 @@ container_images:
 #     insecure: true
 #     download: true
 #     download_dir: /mnt/logscraper/sometenant
+#     file_list: ""
 #     job_name:
 #       - test
 #       - test_new
--- a/ansible/roles/logscraper/tasks/main.yml
+++ b/ansible/roles/logscraper/tasks/main.yml
@ -23,14 +23,6 @@
    group: "{{ logscraper_group }}"
    mode: "0755"

- name: Copy logscraper config file
-  template:
-    src: config.yaml.j2
-    dest: "{{ logscraper_dir }}/config.yaml"
-    owner: "{{ logscraper_user }}"
-    group: "{{ logscraper_group }}"
-    mode: "0640"
-
 - name: Ensure container software is installed
  package:
    name: podman
--- a/ansible/roles/logscraper/tasks/service.yml
+++ b/ansible/roles/logscraper/tasks/service.yml
@ -15,6 +15,24 @@
    mode: '0755'
  register: _start_script

+- name: Generate logscraper download file list
+  template:
+    src: config.yaml.j2
+    dest: "{{ logscraper_dir }}/logscraper-{{ item.tenant }}.yaml"
+    owner: "{{ logscraper_user }}"
+    group: "{{ logscraper_group }}"
+    mode: '0644'
+  register: _config_file
+
+- name: Copy logscraper config file
+  template:
+    src: download-list.yaml.j2
+    dest: "{{ logscraper_dir }}/download-list-{{ item.tenant }}.yaml"
+    owner: "{{ logscraper_user }}"
+    group: "{{ logscraper_group }}"
+    mode: "0640"
+  register: _download_file
+
 - name: Generate systemd unit
  template:
    src: logscraper.service.j2
@ -24,7 +42,7 @@

 - name: Set empty logscraper checkpoint file
  file:
-    path: "{{ item.checkpoint_file | default(logscraper_dir + '/checkpoint') }}"
+    path: "{{ item.checkpoint_file | default(logscraper_dir + '/checkpoint-' + item.tenant) }}"
    state: touch
    owner: "{{ logscraper_user }}"
    group: "{{ logscraper_group }}"
@ -36,7 +54,7 @@
    state: restarted
    daemon-reload: true
    enabled: true
-  when: _start_script.changed
+  when: _start_script.changed or _config_file.changed or _download_file.changed

 - name: Ensure that service is running
  service:
--- a/ansible/roles/logscraper/templates/config.yaml.j2
+++ b/ansible/roles/logscraper/templates/config.yaml.j2
@ -1,248 +1,20 @@
 ---
-# NOTE: In the future, this config will include additional settings like
-# username, password, zuul url etc. required by logscraper and logsender.
-# from https://opendev.org/opendev/base-jobs/src/branch/master/roles/submit-logstash-jobs/defaults/main.yaml
-files:
-  # basic file
-  - name: zuul-info/inventory.yaml
-    tags: []
-  - name: job-output.txt
-    tags:
-      - console
-      - console.html
-  # Devstack
-  - name: controller/logs/grenade.sh_log.txt
-    tags:
-      - console
-      - console.html
-      - grenade
-  - name: controller/logs/devstacklog.txt
-    tags:
-      - console
-      - console.html
-      - devstack
-  - name: controller/logs/devstacklog.txt.summary
-    tags:
-      - console
-      - console.html
-      - devstack_summary
-  - name: controller/logs/syslog.txt
-    tags:
-      - syslog
-  - name: controller/logs/performance.json
-    tags:
-      - performance
-      - devstack
-  # Nova and Placement
-  - name: controller/logs/tempest_log.txt
-    tags:
-      - screen
-      - oslofmt
-      - tempest
-      - devstack
-  - name: controller/logs/screen-n-api.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-n-api-meta.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-n-cond-cell1.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-n-cond-cell2.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-n-novnc-cell1.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-n-novnc-cell2.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-n-cpu.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-n-sch.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-n-super-cond.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-  - name: controller/logs/screen-placement-api.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - placement
-  # Neutron
-  - name: controller/logs/screen-q-agt.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-  - name: controller/logs/screen-q-dhcp.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-  - name: controller/logs/screen-q-l3.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-  - name: controller/logs/screen-q-metering.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-  - name: controller/logs/screen-q-svc.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-  - name: controller/logs/screen-q-ovn-metadata-agent.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-  # Keystone
-  - name: controller/logs/screen-keystone.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - keystone
-  # Cinder
-  - name: controller/logs/screen-c-api.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - cinder
-  - name: controller/logs/screen-c-bak.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - cinder
-  - name: controller/logs/screen-c-sch.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - cinder
-  - name: controller/logs/screen-c-vol.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - cinder
-  # Glance
-  - name: controller/logs/screen-g-api.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - glance
-  # TripleO
-  - name: logs/undercloud/var/log/extra/logstash.txt
-    tags:
-      - console
-      - postci
-      - tripleo
-  - name: logs/undercloud/var/log/extra/errors.txt
-    tags:
-      - console
-      - errors
-      - tripleo
-  - name: logs/undercloud/var/log/tempest/tempest_run.log
-    tags:
-      - tempest
-      - tripleo
-  - name: logs/undercloud/var/log/extra/journal.txt
-    tags:
-      - tempest
-      - errors
-      - tripleo
-  # Multinode log results
-  - name: compute1/logs/syslog.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - syslog
-      - multinode
-  - name: compute1/logs/screen-n-cpu.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - nova
-      - multinode
-  - name: compute1/logs/screen-q-agt.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-      - multinode
-  - name: compute1/logs/screen-q-l3.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-      - multinode
-  - name: compute1/logs/screen-q-dhcp.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - neutron
-      - multinode
-  - name: compute1/logs/screen-c-bak.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - cinder
-      - multinode
-  - name: compute1/logs/screen-c-vol.txt
-    tags:
-      - screen
-      - oslofmt
-      - devstack
-      - cinder
-      - multinode
+zuul_api_url: {{ item['zuul_api_url'].split(', ') }}
+follow: {{ item['follow'] | default(true) }}
+checkpoint_file: {{ item['checkpoint_file'] | default(logscraper_dir + '/checkpoint') }}
+workers: {{ item['logscraper_workers'] | default(1) }}
+max_skipped: {{ item['max_skipped'] | default(1000) }}
+debug: {{ item['debug'] | default(false) }}
+download: {{ item['download'] | default(true) }}
+file_list: {{ logscraper_dir }}/download-list-{{ item['tenant'] }}.yaml
+directory: {{ item['download_dir'] | default('/tmp/logscraper') }}
+wait_time: {{ item['logscraper_wait_time'] | default(120) }}
+insecure: {{ item['insecure'] | default(false) }}
+ca_file: {{ item['custom_ca_crt'] | default('') }}
+#deprecated
+gearman_server: {{ item['gearman_server'] | default('') }}
+gearman_port: {{ item['gearman_port'] | default(4730) }}
+logstash_url: "{{ item['logstash_url'] | default('') }}"
+{% if 'job_name' in item %}
+job_name: {{ item['job_name'].split(', ') }}
+{% endif %}
--- a/ansible/roles/logscraper/templates/download-list.yaml.j2
+++ b/ansible/roles/logscraper/templates/download-list.yaml.j2
--- a/ansible/roles/logscraper/templates/logscraper.sh.j2
+++ b/ansible/roles/logscraper/templates/logscraper.sh.j2
@ -15,43 +15,4 @@
    --volume {{ item.download_dir }}:{{ item.download_dir }}:z \
    {% endif %}
    {{ container_images['logscraper'] }} \
-    /usr/local/bin/logscraper \
-    --zuul-api-url {{ item.zuul_api_url }} \
-    --config {{ logscraper_dir }}/config.yaml \
-    {% if 'job_name' in item %}
-    {% for job in item['job_name'] %}
-    --job-name {{ job }} \
-    {% endfor %}
-    {% endif %}
-    {% if 'gearman_port' in item and 'gearman_server' in item %}
-    --gearman-port {{ item['gearman_port'] }} \
-    --gearman-server {{ item['gearman_server'] }} \
-    {% else %}
-    --download \
-    {% endif %}
-    {% if 'download_dir' in item %}
-    --directory {{ item['download_dir'] }} \
-    {% endif %}
-    {% if 'insecure' in item %}
-    --insecure \
-    {% endif %}
-    --checkpoint-file {{ item.checkpoint_file | default(logscraper_dir + '/checkpoint') }} \
-    {% if 'logstash_url' in item %}
-    --logstash-url {{ item['logstash_url'] }} \
-    {% endif %}
-    {% if 'logscraper_workers' in item %}
-    --workers {{ item['logscraper_workers'] }} \
-    {% endif %}
-    {% if 'max_skipped' in item %}
-    --max-skipped {{ item.max_skipped }} \
-    {% endif %}
-    {% if 'debug' in item %}
-    --debug \
-    {% endif %}
-    {% if 'logscraper_wait_time' in item %}
-    --wait-time {{ item['logscraper_wait_time'] }} \
-    {% endif %}
-    {% if 'custom_ca_crt' in item %}
-    --ca-file {{ item['custom_ca_crt'] }} \
-    {% endif %}
-    --follow
+    /usr/local/bin/logscraper --config {{ logscraper_dir }}/logscraper-{{ item['tenant'] }}.yaml
--- a/ansible/roles/logsender/tasks/service.yml
+++ b/ansible/roles/logsender/tasks/service.yml
@ -6,6 +6,13 @@
    mode: '0755'
  register: _start_script

+- name: Generate logsender configuration file
+  template:
+    src: config.yaml.j2
+    dest: "{{ logscraper_dir }}/logsender-{{ item.tenant }}.yaml"
+    mode: '0644'
+  register: _config_file
+
 - name: Generate systemd unit
  template:
    src: logsender.service.j2
@ -19,7 +26,7 @@
    state: restarted
    daemon-reload: true
    enabled: true
-  when: _start_script.changed
+  when: _start_script.changed or _config_file.changed

 - name: Ensure that service is running
  service:
--- a/ansible/roles/logsender/templates/config.yaml.j2
+++ b/ansible/roles/logsender/templates/config.yaml.j2
@ -0,0 +1,20 @@
+---
+host: {{ item['es_host'] | default('localhost') }}
+port: {{ item['es_port'] | default(9200) }}
+username: {{ item['es_username'] | default('logstash') }}
+password: "{{ item['es_password'] | default('') }}"
+index: {{ item['es_index'] | default('') }}
+index_prefix: {{ item['es_index_prefix'] | default('logstash-') }}
+doc_type: {{ item['doc_type'] | default('_doc') }}
+chunk_size: {{ item['chunk_size'] | default(1500) }}
+skip_debug: {{ item['skip_debug'] | default(true) }}
+keep: {{ item['keep'] | default(false) }}
+debug: {{ item['debug'] | default(false) }}
+wait_time: {{ item['logsender_wait_time'] | default(120) }}
+insecure: {{ item['es_insecure'] | default(false) }}
+ca_file: {{ item['custom_ca_crt'] | default('') }}
+follow: {{ item['follow'] | default(true) }}
+workers: {{ item['logsender_workers'] | default(1) }}
+ignore_es_status: {{ item['ignore_es_status'] | default(false) }}
+directory: {{ item['download_dir'] | default('/tmp/logscraper') }}
+file_list: {{ logscraper_dir }}/download-list-{{ item['tenant'] }}.yaml
--- a/ansible/roles/logsender/templates/logsender.sh.j2
+++ b/ansible/roles/logsender/templates/logsender.sh.j2
@ -13,54 +13,4 @@
    --volume {{ item['custom_ca_crt'] }}:{{ item['custom_ca_crt'] }}:z \
    {% endif %}
    {{ container_images['logsender'] }} \
-    /usr/local/bin/logsender \
-    --config {{ logscraper_dir }}/config.yaml \
-    {% if 'es_host' in item and item['es_host'] %}
-    --host "{{ item['es_host'] }}" \
-    {% endif %}
-    {% if 'es_port' in item and item['es_port'] %}
-    --port "{{ item['es_port'] }}" \
-    {% endif %}
-    {% if 'es_username' in item and item['es_username'] %}
-    --username "{{ item.es_username }}" \
-    {% endif %}
-    {% if 'es_password' in item and item['es_password'] %}
-    --password "{{ item.es_password }}" \
-    {% endif %}
-    {% if 'es_index_prefix' in item and item['es_index_prefix'] %}
-    --index-prefix "{{ item['es_index_prefix'] }}" \
-    {% endif %}
-    {% if 'es_index' in item and item['es_index'] %}
-    --index "{{ item['es_index'] }}" \
-    {% endif %}
-    {% if 'download_dir' in item and item['download_dir'] %}
-    --directory "{{ item['download_dir'] }}" \
-    {% endif %}
-    {% if 'doc_type' in item and item['doc_type'] %}
-    --doc-type "{{ item['doc_type'] }}" \
-    {% endif %}
-    {% if 'es_insecure' in item and item['es_insecure'] %}
-    --insecure \
-    {% endif %}
-    {% if 'logsender_workers' in item and item['logsender_workers'] %}
-    --workers "{{ item['logsender_workers'] }}" \
-    {% endif %}
-    {% if 'chunk_size' in item and item['chunk_size'] %}
-    --chunk-size "{{ item['chunk_size'] }}" \
-    {% endif %}
-    {% if 'keep' in item and item['keep'] %}
-    --keep \
-    {% endif %}
-    {% if 'ignore_es_status' in item and item['ignore_es_status'] %}
-    --ignore-es-status \
-    {% endif %}
-    {% if 'logsender_wait_time' in item %}
-    --wait-time {{ item['logsender_wait_time'] }} \
-    {% endif %}
-    {% if 'custom_ca_crt' in item %}
-    --ca-file {{ item['custom_ca_crt'] }} \
-    {% endif %}
-    {% if 'skip_debug' in item and item['skip_debug'] %}
-    --skip-debug \
-    {% endif %}
-    --follow
+    /usr/local/bin/logsender --config {{ logscraper_dir }}/logsender-{{ item['tenant'] }}.yaml
--- a/logscraper/download-list.yaml.sample
+++ b/logscraper/download-list.yaml.sample
@ -0,0 +1,248 @@
+---
+# NOTE: In the future, this config will include additional settings like
+# username, password, zuul url etc. required by logscraper and logsender.
+# from https://opendev.org/opendev/base-jobs/src/branch/master/roles/submit-logstash-jobs/defaults/main.yaml
+files:
+  # basic file
+  - name: zuul-info/inventory.yaml
+    tags: []
+  - name: job-output.txt
+    tags:
+      - console
+      - console.html
+  # Devstack
+  - name: controller/logs/grenade.sh_log.txt
+    tags:
+      - console
+      - console.html
+      - grenade
+  - name: controller/logs/devstacklog.txt
+    tags:
+      - console
+      - console.html
+      - devstack
+  - name: controller/logs/devstacklog.txt.summary
+    tags:
+      - console
+      - console.html
+      - devstack_summary
+  - name: controller/logs/syslog.txt
+    tags:
+      - syslog
+  - name: controller/logs/performance.json
+    tags:
+      - performance
+      - devstack
+  # Nova and Placement
+  - name: controller/logs/tempest_log.txt
+    tags:
+      - screen
+      - oslofmt
+      - tempest
+      - devstack
+  - name: controller/logs/screen-n-api.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-n-api-meta.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-n-cond-cell1.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-n-cond-cell2.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-n-novnc-cell1.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-n-novnc-cell2.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-n-cpu.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-n-sch.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-n-super-cond.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+  - name: controller/logs/screen-placement-api.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - placement
+  # Neutron
+  - name: controller/logs/screen-q-agt.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+  - name: controller/logs/screen-q-dhcp.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+  - name: controller/logs/screen-q-l3.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+  - name: controller/logs/screen-q-metering.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+  - name: controller/logs/screen-q-svc.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+  - name: controller/logs/screen-q-ovn-metadata-agent.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+  # Keystone
+  - name: controller/logs/screen-keystone.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - keystone
+  # Cinder
+  - name: controller/logs/screen-c-api.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - cinder
+  - name: controller/logs/screen-c-bak.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - cinder
+  - name: controller/logs/screen-c-sch.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - cinder
+  - name: controller/logs/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - cinder
+  # Glance
+  - name: controller/logs/screen-g-api.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - glance
+  # TripleO
+  - name: logs/undercloud/var/log/extra/logstash.txt
+    tags:
+      - console
+      - postci
+      - tripleo
+  - name: logs/undercloud/var/log/extra/errors.txt
+    tags:
+      - console
+      - errors
+      - tripleo
+  - name: logs/undercloud/var/log/tempest/tempest_run.log
+    tags:
+      - tempest
+      - tripleo
+  - name: logs/undercloud/var/log/extra/journal.txt
+    tags:
+      - tempest
+      - errors
+      - tripleo
+  # Multinode log results
+  - name: compute1/logs/syslog.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - syslog
+      - multinode
+  - name: compute1/logs/screen-n-cpu.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - nova
+      - multinode
+  - name: compute1/logs/screen-q-agt.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+      - multinode
+  - name: compute1/logs/screen-q-l3.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+      - multinode
+  - name: compute1/logs/screen-q-dhcp.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - neutron
+      - multinode
+  - name: compute1/logs/screen-c-bak.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - cinder
+      - multinode
+  - name: compute1/logs/screen-c-vol.txt
+    tags:
+      - screen
+      - oslofmt
+      - devstack
+      - cinder
+      - multinode
--- a/logscraper/logscraper.py
+++ b/logscraper/logscraper.py
@ -106,22 +106,17 @@ def requests_get_json(url, verify=True):
 def get_arguments():
    parser = argparse.ArgumentParser(description="Fetch and push last Zuul "
                                     "CI job logs into gearman.")
-    parser.add_argument("--config", help="Logscraper config file",
-                        required=True)
+    parser.add_argument("--config", help="Logscraper config file")
+    parser.add_argument("--file-list", help="File list to download")
    parser.add_argument("--zuul-api-url", help="URL(s) for Zuul API. Parameter"
-                        " can be set multiple times.",
-                        required=True,
-                        action='append')
+                        " can be set multiple times.", action='append')
    parser.add_argument("--job-name", help="CI job name(s). Parameter can be "
                        "set multiple times. If not set it would scrape "
-                        "every latest builds.",
-                        action='append')
+                        "every latest builds.", action='append')
    parser.add_argument("--gearman-server", help="Gearman host addresss")
-    parser.add_argument("--gearman-port", help="Gearman listen port. "
-                        "Defaults to 4730.",
-                        default=4730)
-    parser.add_argument("--follow", help="Keep polling zuul builds",
-                        action="store_true")
+    parser.add_argument("--gearman-port", help="Gearman listen port.")
+    parser.add_argument("--follow", help="Keep polling zuul builds", type=bool,
+                        default=True)
    parser.add_argument("--insecure", help="Skip validating SSL cert",
                        action="store_false")
    parser.add_argument("--checkpoint-file", help="File that will keep "
@ -131,29 +126,47 @@ def get_arguments():
                        "to log processing system. For example: "
                        "logstash.local:9999")
    parser.add_argument("--workers", help="Worker processes for logscraper",
-                        type=int,
-                        default=1)
+                        type=int)
    parser.add_argument("--max-skipped", help="How many job results should be "
                        "checked until last uuid written in checkpoint file "
-                        "is founded",
-                        default=1000)
-    parser.add_argument("--debug", help="Print more information",
-                        action="store_true")
+                        "is founded")
+    parser.add_argument("--debug", help="Print more information", type=bool,
+                        default=False)
    parser.add_argument("--download", help="Download logs and do not send "
-                        "to gearman service",
-                        action="store_true")
+                        "to gearman service")
    parser.add_argument("--directory", help="Directory, where the logs will "
-                        "be stored. Defaults to: /tmp/logscraper",
-                        default="/tmp/logscraper")
+                        "be stored.")
    parser.add_argument("--wait-time", help="Pause time for the next "
-                        "iteration",
-                        type=int,
-                        default=120)
+                        "iteration", type=int)
    parser.add_argument("--ca-file", help="Provide custom CA certificate")
    args = parser.parse_args()
    return args


+def get_config_args(config_path):
+    config_file = load_config(config_path)
+    if config_file:
+        return config_file
+
+
+def parse_args(app_args, config_args):
+    if not config_args:
+        logging.warning("Can not get informations from config files")
+
+    # NOTE: When insecure flag is set as an argument, the value is False,
+    # so if insecure is set to True in config file, it should also be False.
+    if not getattr(app_args, 'insecure') or (
+            'insecure' in config_args and config_args['insecure']):
+        setattr(app_args, 'insecure', False)
+
+    for k, v in config_args.items():
+        # Arguments provided via CLI should have higher priority than
+        # provided in config.
+        if getattr(app_args, k, None) is None:
+            setattr(app_args, k, v)
+    return app_args
+
+
 ###############################################################################
 #                      Configuration of this process                          #
 ###############################################################################
@ -174,7 +187,7 @@ class Config:
            self.filename = "%s-%s" % (self.filename, job_name)

        self.build_cache = BuildCache(self.filename)
-        self.config_file = load_config(args.config)
+        self.config_file = load_config(args.file_list)

    def save(self):
        try:
@ -436,6 +449,8 @@ def load_config(config_path):
            return yaml.safe_load(f)
    except PermissionError:
        logging.critical("Can not open config file %s" % config_path)
+    except FileNotFoundError:
+        logging.critical("Can not find provided config file! %s" % config_path)
    except Exception as e:
        logging.critical("Exception occured on reading config file %s" % e)

@ -689,7 +704,10 @@ def run(args):


 def main():
-    args = get_arguments()
+    app_args = get_arguments()
+    config_args = get_config_args(app_args.config)
+    args = parse_args(app_args, config_args)
+
    setup_logging(args.debug)
    if args.download and args.gearman_server and args.gearman_port:
        logging.critical("Can not use logscraper to send logs to gearman "
--- a/logscraper/logscraper.yaml.sample
+++ b/logscraper/logscraper.yaml.sample
@ -0,0 +1,18 @@
+---
+# logscraper
+zuul_api_url: ['https://zuul.opendev.org/api/tenant/openstack']
+job_name: []
+follow: true
+checkpoint_file: /tmp/logscraper-checkpoint
+workers: 1
+max_skipped: 1000
+debug: false
+download: true
+directory: /tmp/logscraper
+wait_time: 120
+insecure: false
+ca_file: ""
+#deprecated
+gearman_server: ""
+gearman_port: 4730
+logstash_url: ""
--- a/logscraper/logsender.py
+++ b/logscraper/logsender.py
@ -32,6 +32,14 @@ import shutil
 import sys
 import time

+# FIXME: discover why stestr in tox env can not import base lib
+try:
+    from logscraper import get_config_args
+    from logscraper import parse_args
+except ImportError:
+    from logscraper.logscraper import get_config_args
+    from logscraper.logscraper import parse_args
+
 from opensearchpy import exceptions as opensearch_exceptions
 from opensearchpy import helpers
 from opensearchpy import OpenSearch
@ -44,54 +52,34 @@ from ruamel.yaml import YAML
 def get_arguments():
    parser = argparse.ArgumentParser(description="Check log directories "
                                     "and push to the Opensearch service")
-    parser.add_argument("--config", help="Logscraper config file",
-                        required=True)
+    parser.add_argument("--config", help="Logscraper config file")
+    parser.add_argument("--file-list", help="File list to download")
    parser.add_argument("--directory",
                        help="Directory, where the logs will "
-                        "be stored. Defaults to: /tmp/logscraper",
-                        default="/tmp/logscraper")
-    parser.add_argument("--host",
-                        help="Opensearch host",
-                        default='localhost')
-    parser.add_argument("--port",
-                        help="Opensearch port",
-                        type=int,
-                        default=9200)
-    parser.add_argument("--username",
-                        help="Opensearch username",
-                        default='logstash')
+                        "be stored.")
+    parser.add_argument("--host", help="Opensearch host")
+    parser.add_argument("--port", help="Opensearch port", type=int)
+    parser.add_argument("--username", help="Opensearch username")
    parser.add_argument("--password", help="Opensearch user password")
-    parser.add_argument("--index-prefix", help="Prefix for the index. "
-                        "Defaults to logstash-",
-                        default='logstash-')
-    parser.add_argument("--index",
-                        help="Opensearch index. Defaults to: "
-                        "<index-prefix>-YYYY-DD")
+    parser.add_argument("--index-prefix", help="Prefix for the index.")
+    parser.add_argument("--index", help="Opensearch index")
    parser.add_argument("--doc-type", help="Doc type information that will be"
-                        "send to the Opensearch service",
-                        default="_doc")
-    parser.add_argument("--insecure",
-                        help="Skip validating SSL cert",
+                        "send to the Opensearch service")
+    parser.add_argument("--insecure", help="Skip validating SSL cert",
                        action="store_false")
-    parser.add_argument("--follow", help="Keep sending CI logs",
-                        action="store_true")
+    parser.add_argument("--follow", help="Keep sending CI logs", type=bool,
+                        default=True)
    parser.add_argument("--workers", help="Worker processes for logsender",
-                        type=int,
-                        default=1)
-    parser.add_argument("--chunk-size", help="The bulk chunk size",
-                        type=int,
-                        default=1500)
+                        type=int)
+    parser.add_argument("--chunk-size", help="The bulk chunk size", type=int)
    parser.add_argument("--skip-debug", help="Skip messages that contain: "
-                        "DEBUG word",
-                        action="store_true")
+                        "DEBUG word", type=bool, default=True)
    parser.add_argument("--keep", help="Do not remove log directory after",
-                        action="store_true")
-    parser.add_argument("--debug", help="Be more verbose",
-                        action="store_true")
+                        type=bool)
+    parser.add_argument("--debug", help="Be more verbose", type=bool,
+                        default=False)
    parser.add_argument("--wait-time", help="Pause time for the next "
-                        "iteration",
-                        type=int,
-                        default=120)
+                        "iteration", type=int)
    parser.add_argument("--ca-file", help="Provide custom CA certificate")
    args = parser.parse_args()
    return args
@ -355,7 +343,7 @@ def send(ready_directory, args, directory, index):

    for build_file in build_files:
        fields = copy.deepcopy(es_fields)
-        file_name, file_tags = get_file_info(args.config, build_file)
+        file_name, file_tags = get_file_info(args.file_list, build_file)
        fields["filename"] = build_file
        fields["log_url"] = (fields["log_url"] + file_name if fields[
            "log_url"] else file_name)
@ -469,7 +457,9 @@ def run(args):


 def main():
-    args = get_arguments()
+    app_args = get_arguments()
+    config_args = get_config_args(app_args.config)
+    args = parse_args(app_args, config_args)
    setup_logging(args.debug)
    while True:
        run(args)
--- a/logscraper/logsender.yaml.sample
+++ b/logscraper/logsender.yaml.sample
@ -0,0 +1,18 @@
+---
+directory: /tmp/logscraper
+host: localhost
+port: 9200
+username: logstash
+password: ""
+index_prefix: "logstash-"
+index: ""
+doc_type: "_doc"
+chunk_size: 1500
+skip_debug: true
+keep: false
+debug: false
+wait_time: 120
+insecure: false
+ca_file: ""
+follow: true
+workers: 1
--- a/logscraper/tests/test_logscraper.py
+++ b/logscraper/tests/test_logscraper.py
@ -149,7 +149,8 @@ class FakeArgs(object):
                 checkpoint_file=None, ignore_checkpoint=None,
                 logstash_url=None, workers=None, max_skipped=None,
                 job_name=None, download=None, directory=None,
-                 config=None, wait_time=None, ca_file=None):
+                 config=None, wait_time=None, ca_file=None,
+                 file_list=None):

        self.zuul_api_url = zuul_api_url
        self.gearman_server = gearman_server
@ -167,6 +168,7 @@ class FakeArgs(object):
        self.config = config
        self.wait_time = wait_time
        self.ca_file = ca_file
+        self.file_list = file_list


 class TestScraper(base.TestCase):
--- a/logscraper/tests/test_logsender.py
+++ b/logscraper/tests/test_logsender.py
@ -273,7 +273,7 @@ class FakeArgs(object):
                 username=None, password=None, index_prefix=None, index=None,
                 doc_type=None, insecure=None, follow=None, workers=None,
                 chunk_size=None, skip_debug=None, keep=None, debug=None,
-                 wait_time=None):
+                 wait_time=None, file_list=None):

        self.config = config
        self.directory = directory
@ -292,6 +292,7 @@ class FakeArgs(object):
        self.keep = keep
        self.debug = debug
        self.wait_time = wait_time
+        self.file_list = file_list


 class TestSender(base.TestCase):
--- a/setup.py
+++ b/setup.py
@ -18,5 +18,7 @@ setuptools.setup(
    setup_requires=['pbr'],
    pbr=True,
    include_package_data=True,
-    package_data={'logscraper': ['config.yaml.sample']}
+    package_data={'logscraper': ['download-list.yaml.sample',
+                                 'logscraper.yaml.sample',
+                                 'logsender.yaml.sample']}
 )