diff --git a/.gitignore b/.gitignore index c4f2973..f54033f 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,6 @@ releasenotes/build # ansible retry files *.retry ansible_role_collect_logs.egg-info + +# buit collection +*.tar.gz diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a81a82..3acb4ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,16 @@ --- repos: - repo: https://github.com/PyCQA/isort - rev: 5.8.0 + rev: 5.9.3 hooks: - id: isort - repo: https://github.com/python/black.git - rev: 20.8b1 + rev: 21.7b0 hooks: - id: black language_version: python3 - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - id: end-of-file-fixer - id: trailing-whitespace @@ -19,10 +19,8 @@ repos: - id: check-executables-have-shebangs - id: check-merge-conflict - id: debug-statements - - id: check-yaml - files: .*\.(yaml|yml)$ - repo: https://gitlab.com/pycqa/flake8.git - rev: 3.9.1 + rev: 3.9.2 hooks: - id: flake8 additional_dependencies: @@ -30,7 +28,7 @@ repos: - flake8-black>=0.1.1 language_version: python3 - repo: https://github.com/ansible/ansible-lint.git - rev: v5.0.7 + rev: v5.1.2 hooks: - id: ansible-lint always_run: true @@ -38,7 +36,9 @@ repos: # results when called with individual files. # https://github.com/ansible/ansible-lint/issues/611 verbose: true - entry: env ANSIBLE_LIBRARY=./library ansible-lint --force-color -v . + additional_dependencies: + - ansible-core + - yamllint - repo: https://github.com/openstack-dev/bashate.git rev: 2.0.0 hooks: diff --git a/.yamllint b/.yamllint deleted file mode 100644 index 689b7b5..0000000 --- a/.yamllint +++ /dev/null @@ -1,7 +0,0 @@ ---- -extends: default - -rules: - line-length: - # matches hardcoded 160 value from ansible-lint - max: 160 diff --git a/MANIFEST.in b/MANIFEST.in index 0abd561..000493d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,11 @@ global-exclude __pycache__ +exclude .benchmarks +exclude .eggs +exclude .mypy_cache +exclude .pytest_cache +exclude .quickstart +exclude .tox exclude infrared_plugin -exclude zuul.d -exclude test-playbooks exclude plugins +exclude test-playbooks +exclude zuul.d diff --git a/galaxy.yml b/galaxy.yml index 6a51824..c027102 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -7,19 +7,39 @@ authors: readme: README.rst build_ignore: + - "**/.mypy_cache" + - "**/.pytest_cache" + - "*.egg-info" + - .DS_Store - .ansible + - .benchmarks - .cache + - .eggs + - .envrc - .github - .gitignore - - .pytest_cache - - .vscode - - .tox - - dist - - tox.ini - - .eggs + - .gitreview - .mypy_cache - - "*.egg-info" - - modules - - module_utils + - .pre-commit-config.yaml + - .pytest_cache + - .quickstart + - .tox + - .vscode + - .yamllint + - ansible.cfg + - bindep.txt + - build + - dist + - docs/source/_build - infrared_plugin + - module_utils + - modules + - pyproject.toml + - report.html + - roles/collect_logs/library - scripts + - setup.cfg + - setup.py + - test-playbooks + - tox.ini + - zuul.d diff --git a/requirements.yml b/requirements.yml new file mode 100644 index 0000000..e30393e --- /dev/null +++ b/requirements.yml @@ -0,0 +1,2 @@ +collections: + - name: ansible.posix # needed by synchronize diff --git a/roles/collect_logs/files/collect-container-logs.sh b/roles/collect_logs/files/collect-container-logs.sh index 852e578..600ffe2 100755 --- a/roles/collect_logs/files/collect-container-logs.sh +++ b/roles/collect_logs/files/collect-container-logs.sh @@ -76,12 +76,12 @@ for cont in $(${engine} ps | awk '{print $NF}' | grep -v NAMES); do # Capture rpms updated from more recent repos update_repos="gating delorean-current" if ls /etc/yum.repos.d/*-component.repo 1> /dev/null 2>&1; then - component_name=$(cat $(ls /etc/yum.repos.d/*-component.repo) | grep "name=" | sed "s/name=//g") + component_name=$(cat /etc/yum.repos.d/*-component.repo | grep "name=" | sed "s/name=//g") update_repos="${update_repos} ${component_name}" fi echo "*** ${cont} rpm update info ***" >> "$BASE_CONTAINER_EXTRA/container_updates_info.log" for repo in $update_repos; do - cat "$INFO_DIR/${engine}_info.log" | grep "@${repo}" >> "$BASE_CONTAINER_EXTRA/container_updates_info.log" + grep "@${repo}" "$INFO_DIR/${engine}_info.log" >> "$BASE_CONTAINER_EXTRA/container_updates_info.log" done; # NOTE(flaper87): This should go away. Services should be diff --git a/roles/collect_logs/molecule/default/converge.yml b/roles/collect_logs/molecule/default/converge.yml index d0f9eac..f42892b 100644 --- a/roles/collect_logs/molecule/default/converge.yml +++ b/roles/collect_logs/molecule/default/converge.yml @@ -8,6 +8,7 @@ file: path: "{{ local_working_dir }}" state: directory + mode: 0755 # Assure we do not have leftovers from previous runs there, not needed in production. - name: Remove collected_files folder @@ -88,6 +89,7 @@ include_role: name: collect_logs - - debug: + - name: Display stats + debug: msg: | Collected files should be under {{ local_working_dir }}/collected_files diff --git a/roles/collect_logs/molecule/default/prepare.yml b/roles/collect_logs/molecule/default/prepare.yml index c123eef..8f71770 100644 --- a/roles/collect_logs/molecule/default/prepare.yml +++ b/roles/collect_logs/molecule/default/prepare.yml @@ -9,6 +9,7 @@ content: | 2000-00-00T00:00:00.000Z | this is sparta dest: /var/log/bootstrap-subnodes.log + mode: 0644 - name: Remove /var/log/extra/logstash.txt file: diff --git a/roles/collect_logs/molecule/default/verify.yml b/roles/collect_logs/molecule/default/verify.yml index 74d7352..00c174f 100644 --- a/roles/collect_logs/molecule/default/verify.yml +++ b/roles/collect_logs/molecule/default/verify.yml @@ -4,5 +4,5 @@ tasks: - name: Check logstash.txt contains expected data - shell: grep -q "this is sparta" /var/log/extra/logstash.txt + command: grep -q "this is sparta" /var/log/extra/logstash.txt changed_when: false diff --git a/roles/collect_logs/molecule/infrared/converge.yml b/roles/collect_logs/molecule/infrared/converge.yml index 8b168c8..4806769 100644 --- a/roles/collect_logs/molecule/infrared/converge.yml +++ b/roles/collect_logs/molecule/infrared/converge.yml @@ -30,6 +30,7 @@ file: path: "{{ infrared_location }}/infrared_plugin" state: directory + mode: 0755 - name: "Copy ansible-role-collect-logs to test host" synchronize: @@ -56,6 +57,7 @@ file: path: "{{ item }}" state: touch + mode: 0644 with_items: - "/tmp/dummy.log" - "/tmp/append.log" diff --git a/roles/collect_logs/molecule/infrared/molecule.yml b/roles/collect_logs/molecule/infrared/molecule.yml index 0056544..b22c6ec 100644 --- a/roles/collect_logs/molecule/infrared/molecule.yml +++ b/roles/collect_logs/molecule/infrared/molecule.yml @@ -45,3 +45,5 @@ scenario: - verify - cleanup - destroy +markers: + - xfail # https://projects.engineering.redhat.com/browse/RHOSINFRA-4174 diff --git a/roles/collect_logs/molecule/infrared/verify.yml b/roles/collect_logs/molecule/infrared/verify.yml index 2a82f66..1cfc2c9 100644 --- a/roles/collect_logs/molecule/infrared/verify.yml +++ b/roles/collect_logs/molecule/infrared/verify.yml @@ -24,7 +24,7 @@ - name: Get directory with collected log files stat: - path: "{{ infrared_location}}/collected_files_test" + path: "{{ infrared_location }}/collected_files_test" register: collected_files_dir diff --git a/roles/collect_logs/molecule/sova/prepare.yml b/roles/collect_logs/molecule/sova/prepare.yml index d8d6c3a..af71df8 100644 --- a/roles/collect_logs/molecule/sova/prepare.yml +++ b/roles/collect_logs/molecule/sova/prepare.yml @@ -7,9 +7,11 @@ file: path: '{{ ansible_user_dir }}/workspace/logs/' state: directory + mode: 0755 - name: Create a sample console file copy: content: | No valid host was found. There are not enough hosts dest: '{{ ansible_user_dir }}/workspace/logs/quickstart_install.log' + mode: 0644 diff --git a/roles/collect_logs/tasks/collect.yml b/roles/collect_logs/tasks/collect.yml index 9a3c661..6294425 100644 --- a/roles/collect_logs/tasks/collect.yml +++ b/roles/collect_logs/tasks/collect.yml @@ -11,11 +11,13 @@ file: dest: /var/log/extra state: directory + mode: 0755 - name: Create rsync filter file template: src: "odl_extra_logs.j2" dest: "/tmp/odl_extra_logs.sh" + mode: 0644 - name: Determine commands to run run_once: true @@ -90,11 +92,13 @@ file: dest: "/tmp/{{ inventory_hostname }}" state: directory + mode: 0755 - name: Create rsync filter file template: src: "rsync-filter.j2" dest: "/tmp/{{ inventory_hostname }}-rsync-filter" + mode: 0644 when: artcl_rsync_collect_list|bool # This task needs to be finished before generating find list of files @@ -166,7 +170,7 @@ tags: - skip_ansible_lint -- name: Create tar archive of logs for faster copying +- name: Create tar archive of logs for faster copying # noqa: command-instead-of-module shell: cmd: tar cf {{ inventory_hostname }}.tar {{ inventory_hostname }}; chdir: /tmp @@ -183,7 +187,7 @@ file: path: "/tmp/{{ inventory_hostname }}" state: absent - ignore_errors: true + ignore_errors: true # noqa ignore-errors - name: Extract the logs archive unarchive: diff --git a/roles/collect_logs/tasks/collect/container.yml b/roles/collect_logs/tasks/collect/container.yml index cf61032..2fb3b9b 100644 --- a/roles/collect_logs/tasks/collect/container.yml +++ b/roles/collect_logs/tasks/collect/container.yml @@ -12,12 +12,15 @@ when: odl_container_enabled.rc != 0 - - name: check if ODL is enabled via rpm + - name: check if ODL is enabled via rpm # noqa: command-instead-of-module shell: rpm -qa | grep opendaylight register: odl_rpm_enabled - name: Create ODL log directory - file: dest="{{ odl_extra_log_dir }}" state=directory + file: + dest: "{{ odl_extra_log_dir }}" + state: directory + mode: 0755 when: (odl_rpm_enabled.rc == 0) or (odl_container_enabled.rc == 0) - name: Collect OVS outputs for ODL @@ -34,6 +37,7 @@ copy: src: collect-container-logs.sh dest: /tmp/collect-container-logs.sh + mode: 0755 - name: Run container logs collection with timeout command: >- diff --git a/roles/collect_logs/tasks/create-docs.yml b/roles/collect_logs/tasks/create-docs.yml index 224faee..bb6b53e 100644 --- a/roles/collect_logs/tasks/create-docs.yml +++ b/roles/collect_logs/tasks/create-docs.yml @@ -11,14 +11,14 @@ "{{ artcl_collect_dir }}/undercloud/home/{{ undercloud_user }}/{{ item }}.sh" > \ "{{ artcl_docs_source_dir }}/{{ item }}.rst" with_items: "{{ artcl_create_docs_payload.included_deployment_scripts }}" - ignore_errors: true + ignore_errors: true # noqa: ignore-errors changed_when: true - name: Fetch static rST docs to include in output docs shell: > cp "{{ artcl_docs_source_dir }}/../static/{{ item }}.rst" "{{ artcl_docs_source_dir }}" with_items: "{{ artcl_create_docs_payload.included_static_docs }}" - ignore_errors: true + ignore_errors: true # noqa: ignore-errors changed_when: true - name: Generate fresh index.rst for Sphinx @@ -26,11 +26,13 @@ src: index.rst.j2 dest: "{{ artcl_docs_source_dir }}/index.rst" force: true + mode: 0644 - name: Ensure docs dir exists file: path: "{{ artcl_collect_dir }}/docs" state: directory + mode: 0755 - name: Build docs with Sphinx shell: > @@ -39,5 +41,5 @@ 2>&1 {{ timestamper_cmd }} > {{ artcl_collect_dir }}/docs/sphinx_build.log args: executable: /bin/bash - ignore_errors: true + ignore_errors: true # noqa: ignore-errors changed_when: true diff --git a/roles/collect_logs/tasks/publish.yml b/roles/collect_logs/tasks/publish.yml index cd9c017..75f7c8a 100644 --- a/roles/collect_logs/tasks/publish.yml +++ b/roles/collect_logs/tasks/publish.yml @@ -61,7 +61,7 @@ with_items: - "{{ local_working_dir }}/*.sh" - "{{ local_working_dir }}/*.log" - ignore_errors: true + ignore_errors: true # noqa: ignore-errors changed_when: true # the doc footer for logging has been removed. @@ -71,7 +71,7 @@ if [ -f {{ artcl_readme_path }} ]; then cp {{ artcl_readme_path }} {{ artcl_readme_file }}; fi; - ignore_errors: true + ignore_errors: true # noqa: ignore-errors changed_when: true - name: Rename compressed text based files to end with txt.gz extension @@ -98,8 +98,8 @@ # This is where upload starts, no local change after this line # All tags after this line are marked with "notest" to allow testing (molecule) in absence of access to log servers. -- name: upload to the artifact server using pubkey auth - shell: > +- name: upload to the artifact server using pubkey auth # noqa: command-instead-of-module + command: > rsync -av --quiet -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {{ artcl_report_server_key|default('') }}" {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ artcl_build_tag }} @@ -110,10 +110,10 @@ when: artcl_use_rsync|bool and not artcl_rsync_use_daemon|bool tags: notest -- name: upload to the artifact server using password auth +- name: upload to the artifact server using password auth # noqa: command-instead-of-module environment: RSYNC_PASSWORD: "{{ artcl_rsync_password | default(omit) }}" - shell: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ artcl_build_tag }} # noqa 305 + command: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ artcl_build_tag }} async: "{{ artcl_publish_timeout }}" poll: 15 retries: 5 @@ -149,5 +149,6 @@ template: src: full_logs.html.j2 dest: "{{ artcl_collect_dir }}/full_logs.html" + mode: 0644 when: artcl_env != 'tripleo-ci' tags: notest diff --git a/roles/collect_logs/tasks/publish_ara.yml b/roles/collect_logs/tasks/publish_ara.yml index 3d0523e..b3b8e1a 100644 --- a/roles/collect_logs/tasks/publish_ara.yml +++ b/roles/collect_logs/tasks/publish_ara.yml @@ -21,11 +21,11 @@ {{ local_working_dir }}/bin/ara generate html {{ local_working_dir }}/ara_oooq_oc; {{ local_working_dir }}/bin/ara task list --all -f json > {{ artcl_collect_dir }}/ara.oooq.oc.json; cp -r {{ local_working_dir }}/ara_oooq_oc {{ artcl_collect_dir }}/; - ignore_errors: true + ignore_errors: true # noqa: ignore-errors environment: ARA_DATABASE: 'sqlite:///{{ ara_overcloud_db_path }}' -- name: Copy ara files to ara-report directories +- name: Copy ara files to ara-report directories # noqa: deprecated-command-syntax shell: | mkdir -p {{ artcl_collect_dir }}/{{ item.dir }}/ara-report; cp {{ item.file }} {{ artcl_collect_dir }}/{{ item.dir }}/ara-report/ansible.sqlite; diff --git a/roles/collect_logs/tasks/publish_ara_influxdb.yml b/roles/collect_logs/tasks/publish_ara_influxdb.yml index 93e56ce..073b12a 100644 --- a/roles/collect_logs/tasks/publish_ara_influxdb.yml +++ b/roles/collect_logs/tasks/publish_ara_influxdb.yml @@ -17,6 +17,7 @@ measurement: "{{ influxdb_measurement }}" data_file: "{{ influxdb_data_file_path }}" only_successful_tasks: "{{ influxdb_only_successful_tasks }}" + no_log: true - name: Get ARA json data for undercloud become: true @@ -40,6 +41,7 @@ standard_fields: false longest_tasks: 15 when: ara_root_data.stdout != "[]" + no_log: true - name: Get ARA json data for overcloud shell: "{{ local_working_dir }}/bin/ara result list --all -f json" # noqa 305 @@ -64,3 +66,4 @@ standard_fields: false longest_tasks: 15 when: ara_oc_data.stdout != "[]" + no_log: true diff --git a/tests/sanity/ignore-2.10.txt b/tests/sanity/ignore-2.10.txt index ba428dd..83ff2bf 100644 --- a/tests/sanity/ignore-2.10.txt +++ b/tests/sanity/ignore-2.10.txt @@ -2,4 +2,3 @@ plugins/modules/ara_graphite.py validate-modules:missing-gplv3-license plugins/modules/ara_influxdb.py validate-modules:missing-gplv3-license plugins/modules/flatten_nested_dict.py validate-modules:missing-gplv3-license plugins/modules/sova.py validate-modules:missing-gplv3-license -docs/static/env-setup-virt.rst rstcheck diff --git a/tests/sanity/ignore-2.11.txt b/tests/sanity/ignore-2.11.txt new file mode 120000 index 0000000..0c39ad8 --- /dev/null +++ b/tests/sanity/ignore-2.11.txt @@ -0,0 +1 @@ +ignore-2.10.txt \ No newline at end of file diff --git a/tox.ini b/tox.ini index cdc2bdc..ed2391c 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ minversion = 3.4.0 envlist = docs, linters, molecule skipsdist = True requires = - tox-ansible >= 1.0.5 + tox-ansible >= 1.5.3 [testenv] usedevelop = True @@ -56,17 +56,25 @@ setenv = ANSIBLE_SHOW_CUSTOM_STATS=1 PYTHONPATH={env:PYTHONPATH:}:library # failsafe mechanism to avoid accidental disablement of tests - # 3 molecule tests expected - PYTEST_REQPASS=3 + # 2/3 molecule scenarios are expected to pass + PYTEST_REQPASS=2 deps = ansible>=2.9,<2.10 - molecule[test,docker]>=3.2.2,<3.3 # MIT - pytest-molecule>=1.3.4 + molecule[test,docker]>=3.3.2,<3.4 # MIT + pytest-molecule>=1.4.0 pytest-plus # provides support for PYTEST_REQPASS commands = + ansible-galaxy collection install community.docker python -m pytest --color=yes --html={envlogdir}/reports.html --self-contained-html {tty:-s} -k molecule {posargs} - [testenv:ansible] description = Used as base for all tox-ansible environments -basepython = python3.6 + +# https://github.com/ansible-community/tox-ansible/issues/96 +[testenv:sanity] +usedevelop = False +deps = + +[testenv:units] +usedevelop = False +deps =