From dc1de268aafba335025abfb6806883cc64dbb5b1 Mon Sep 17 00:00:00 2001 From: Doug Goldstein Date: Sun, 14 Dec 2025 21:18:41 -0600 Subject: [PATCH] feat: build charts into sub dirs As outlined in https://lists.opendev.org/archives/list/service-discuss@lists.opendev.org/thread/VTMDDVSPM5HRUYWAATNMZOILT5OE57VR/ the current structure of building all the charts into one directory is causing issues on the opendev infra due too many entries in one directory. Switch away from using a Makefile to using an Ansible role to build each chart and then use chart-testing to identify the charts that need to be rebuilt and lastly build them build put the output into a subdir matching the chart name. Change-Id: I61f11950ba381c7897eb6bfff05a508ca4db9f06 Signed-off-by: Doug Goldstein Signed-off-by: Vladimir Kozhukalov --- playbooks/build-chart.yaml | 38 +++++++++-- playbooks/lint.yaml | 22 +++---- playbooks/osh-infra-bandit.yaml | 19 ------ playbooks/publish/post.yaml | 63 +++++++++++++++++-- roles/chart-testing/README.rst | 19 ++++++ roles/chart-testing/defaults/main.yaml | 3 + roles/chart-testing/tasks/main.yaml | 6 ++ roles/ensure-chart-testing/README.rst | 17 +++++ roles/ensure-chart-testing/defaults/main.yaml | 5 ++ roles/ensure-chart-testing/tasks/main.yaml | 38 +++++++++++ roles/osh-bandit/tasks/main.yaml | 29 +++++++-- zuul.d/base.yaml | 5 +- zuul.d/project.yaml | 2 + 13 files changed, 215 insertions(+), 51 deletions(-) delete mode 100644 playbooks/osh-infra-bandit.yaml create mode 100644 roles/chart-testing/README.rst create mode 100644 roles/chart-testing/defaults/main.yaml create mode 100644 roles/chart-testing/tasks/main.yaml create mode 100644 roles/ensure-chart-testing/README.rst create mode 100644 roles/ensure-chart-testing/defaults/main.yaml create mode 100644 roles/ensure-chart-testing/tasks/main.yaml diff --git a/playbooks/build-chart.yaml b/playbooks/build-chart.yaml index 3af1c208b0..beea3f165b 100644 --- a/playbooks/build-chart.yaml +++ b/playbooks/build-chart.yaml @@ -13,19 +13,45 @@ - hosts: all roles: - - ensure-python - - ensure-pip - ensure-helm + - ensure-chart-testing tasks: - name: Install reno pip: name: reno>=4.1.0 - extra_args: "--ignore-installed" - become: yes + virtualenv: "{{ virtualenv }}" + virtualenv_command: python3 -m venv - - name: make all + - name: Get list of changed charts + shell: "ct list-changed --target-branch master --since {{ zuul.oldrev | default('HEAD~1') }} --chart-dirs . 2>/dev/null" + args: + chdir: "{{ zuul.project.src_dir }}" + register: changed_charts_output + changed_when: false + + - name: Parse changed charts + set_fact: + changed_charts: "{{ changed_charts_output.stdout_lines }}" + + - name: Display changed charts + debug: + msg: "Changed charts: {{ changed_charts }}" + + - name: Build each changed chart make: chdir: "{{ zuul.project.src_dir }}" - target: all + target: "{{ item }}" + params: + PYTHON: "{{ virtualenv }}/bin/python" + BASE_VERSION: "{{ base_version }}" + loop: "{{ changed_charts }}" + when: changed_charts | length > 0 + + - name: Move chart packages to subdirectories + shell: | + mkdir -p {{ zuul.project.src_dir }}/{{ item }} + mv {{ zuul.project.src_dir }}/{{ item }}-*.tgz {{ zuul.project.src_dir }}/{{ item }}/ + loop: "{{ changed_charts }}" + when: changed_charts | length > 0 ... diff --git a/playbooks/lint.yaml b/playbooks/lint.yaml index b535d9adad..cf071087db 100644 --- a/playbooks/lint.yaml +++ b/playbooks/lint.yaml @@ -26,29 +26,25 @@ work_dir: "{{ zuul.project.src_dir }}" tasks: - - name: Install reno - pip: - name: reno>=4.1.0 - extra_args: "--ignore-installed" - become: yes - - # - name: make all - # make: - # chdir: "{{ work_dir }}" - # target: all - - name: Install yamllint - shell: pip3 install -U yq yamllint - become: yes + pip: + name: + - yq + - yamllint + virtualenv: "{{ virtualenv }}" + virtualenv_command: python3 -m venv - name: Run yamllint shell: | cat > /tmp/yamllint.sh </dev/null" + args: + chdir: "{{ zuul.project.src_dir }}" + register: changed_charts_output + changed_when: false + + - name: Parse changed charts + set_fact: + changed_charts: "{{ changed_charts_output.stdout_lines }}" + + - name: Display changed charts + debug: + msg: "Changed charts to publish: {{ changed_charts }}" + - name: Download current index register: _get_url failed_when: _get_url.status_code not in (200, 404) @@ -30,26 +45,62 @@ when: _get_url.status_code == 200 shell: helm repo index {{ zuul.project.src_dir }} --merge {{ zuul.project.src_dir }}/index.yaml --url https://tarballs.opendev.org/{{ zuul.project.name }} + - name: Cat updated index + shell: cat {{ zuul.project.src_dir }}/index.yaml + register: index_content + changed_when: false + + - name: Display updated index + debug: + msg: "{{ index_content.stdout }}" + - name: Ensure artifact directory exists file: path: "{{ zuul.executor.work_root }}/artifacts/" state: directory delegate_to: localhost - - name: Gather the artifacts + - name: Ensure chart subdirectories exist in artifacts + file: + path: "{{ zuul.executor.work_root }}/artifacts/{{ item }}" + state: directory + delegate_to: localhost + loop: "{{ changed_charts }}" + when: changed_charts | length > 0 + + - name: Gather packaged charts from changed chart directories find: file_type: file paths: "{{ zuul.project.src_dir }}" - patterns: "*.tar.gz,*.tgz,index.yaml" - register: result + patterns: "{{ item }}-*.tgz" + recurse: true + register: chart_packages + loop: "{{ changed_charts }}" + when: changed_charts | length > 0 - - name: Update Helm repository + - name: Display chart tarballs to be published + debug: + msg: "src: {{ item.1.path }} dest: {{ zuul.executor.work_root }}/artifacts/{{ item.0.item }}/" + loop: "{{ chart_packages.results | subelements('files', skip_missing=True) }}" + when: changed_charts | length > 0 + + - name: Copy packaged charts to artifacts preserving directory structure synchronize: mode: pull - src: "{{ item.path }}" + src: "{{ item.1.path }}" + dest: "{{ zuul.executor.work_root }}/artifacts/{{ item.0.item }}/" + verify_host: true + owner: no + group: no + loop: "{{ chart_packages.results | subelements('files', skip_missing=True) }}" + when: changed_charts | length > 0 + + - name: Copy index.yaml to artifacts + synchronize: + mode: pull + src: "{{ zuul.project.src_dir }}/index.yaml" dest: "{{ zuul.executor.work_root }}/artifacts/" verify_host: true owner: no group: no - with_items: "{{ result.files }}" ... diff --git a/roles/chart-testing/README.rst b/roles/chart-testing/README.rst new file mode 100644 index 0000000000..087776bc0e --- /dev/null +++ b/roles/chart-testing/README.rst @@ -0,0 +1,19 @@ +Run chart-testing (for helm charts) + +**Role Variables** + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + The location of the main working directory of the job. + +.. zuul:rolevar:: chart_testing_options + :default: --validate-maintainers=false --check-version-increment=false + + Arguments passed to chart testing. + + The defaults are suitable for a Zuul environment because + `validate-maintainers` requires a valid git remote (which is not + present in Zuul) and `check-version-increment` requires each commit + to have a new version; Zuul users are expected to set the version + when tagging/publishing a release. diff --git a/roles/chart-testing/defaults/main.yaml b/roles/chart-testing/defaults/main.yaml new file mode 100644 index 0000000000..314bb747b7 --- /dev/null +++ b/roles/chart-testing/defaults/main.yaml @@ -0,0 +1,3 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +chart_testing_options: --validate-maintainers=false --check-version-increment=false +virtualenv: "{{ ansible_user_dir }}/venv" diff --git a/roles/chart-testing/tasks/main.yaml b/roles/chart-testing/tasks/main.yaml new file mode 100644 index 0000000000..cf283bc98c --- /dev/null +++ b/roles/chart-testing/tasks/main.yaml @@ -0,0 +1,6 @@ +- name: Run chart-testing + shell: | + source "{{ virtualenv }}/bin/activate" + ct lint {{ chart_testing_options }} + args: + chdir: "{{ zuul_work_dir }}" diff --git a/roles/ensure-chart-testing/README.rst b/roles/ensure-chart-testing/README.rst new file mode 100644 index 0000000000..2b2e6533d1 --- /dev/null +++ b/roles/ensure-chart-testing/README.rst @@ -0,0 +1,17 @@ +Ensure chart-testing is installed + +**Role Variables** + +.. zuul:rolevar:: chart_testing_version + + Version of chart-testing to install. + +.. zuul:rolevar:: ensure_chart_testing_repo_name_helm_chart + :default: https://github.com/helm/chart-testing/releases/download + + The root location to get the chart testing helm chart. + +.. zuul:rolevar:: ensure_chart_testing_repo_name_config + :default: https://raw.githubusercontent.com/helm/chart-testing + + The root location to get the chart testing configuration files. diff --git a/roles/ensure-chart-testing/defaults/main.yaml b/roles/ensure-chart-testing/defaults/main.yaml new file mode 100644 index 0000000000..831a8c2249 --- /dev/null +++ b/roles/ensure-chart-testing/defaults/main.yaml @@ -0,0 +1,5 @@ +--- +chart_testing_version: 2.4.0 +ensure_chart_testing_repo_name_helm_chart: "https://github.com/helm/chart-testing/releases/download" +ensure_chart_testing_repo_name_config: "https://raw.githubusercontent.com/helm/chart-testing" +virtualenv: "{{ ansible_user_dir }}/venv" \ No newline at end of file diff --git a/roles/ensure-chart-testing/tasks/main.yaml b/roles/ensure-chart-testing/tasks/main.yaml new file mode 100644 index 0000000000..d86d8cb88f --- /dev/null +++ b/roles/ensure-chart-testing/tasks/main.yaml @@ -0,0 +1,38 @@ +--- +- name: Install pip + include_role: + name: ensure-pip + +- name: Install Python dependencies + become: false + pip: + name: + - yamale + - yamllint + virtualenv: "{{ virtualenv }}" + virtualenv_command: python3 -m venv + +- name: Install chart-testing + become: true + unarchive: + remote_src: true + src: "{{ ensure_chart_testing_repo_name_helm_chart }}/v{{ chart_testing_version }}/chart-testing_{{ chart_testing_version }}_linux_amd64.tar.gz" + dest: /usr/local/bin + +- name: Setup /etc/ct + become: true + file: + path: /etc/ct + state: directory + mode: 0755 + +- name: Install configuration files + become: true + get_url: + url: "{{ ensure_chart_testing_repo_name_config }}/v{{ chart_testing_version }}/etc/{{ zj_item }}" + dest: "/etc/ct/{{ zj_item }}" + loop: + - chart_schema.yaml + - lintconf.yaml + loop_control: + loop_var: zj_item diff --git a/roles/osh-bandit/tasks/main.yaml b/roles/osh-bandit/tasks/main.yaml index ad4e59aff1..bbd1ae70ff 100644 --- a/roles/osh-bandit/tasks/main.yaml +++ b/roles/osh-bandit/tasks/main.yaml @@ -15,16 +15,29 @@ include_role: name: ensure-helm +- name: Install binary packages + become: true + apt: + name: + - jq + state: present + update_cache: yes + - name: Install yq bandit - shell: | - sudo -H pip3 install --upgrade yq bandit=={{ bandit_version }} setuptools - args: - chdir: "{{ work_dir }}" + pip: + name: + - yq + - bandit=={{ bandit_version }} + - setuptools + - pbr + virtualenv: "{{ virtualenv }}" + virtualenv_command: python3 -m venv - name: Template out python files shell: | set -xe; - make all + source "{{ virtualenv }}/bin/activate" + make all SKIP_CHANGELOG=1 mkdir -p python-files EXCLUDES="helm-toolkit doc tests tools logs tmp roles playbooks releasenotes zuul.d python-files" DIRS=`ls -d */ | cut -f1 -d'/'` @@ -42,9 +55,13 @@ done args: chdir: "{{ work_dir }}" + executable: /bin/bash - name: Run bandit against python files - shell: bandit -r ./python-files + shell: | + source "{{ virtualenv }}/bin/activate" + bandit -r ./python-files -s B404,B603 args: chdir: "{{ work_dir }}" + executable: /bin/bash ... diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 211ae16e3f..0c09a202cc 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -14,7 +14,7 @@ - job: name: openstack-helm-linter run: playbooks/lint.yaml - nodeset: openstack-helm-1node-ubuntu_jammy + nodeset: openstack-helm-1node-ubuntu_noble required-projects: - openstack/openstack-helm irrelevant-files: @@ -29,6 +29,7 @@ - job: name: openstack-helm-bandit + nodeset: openstack-helm-1node-ubuntu_noble roles: - zuul: openstack/openstack-helm - zuul: zuul/zuul-jobs @@ -51,6 +52,8 @@ required-projects: - openstack/openstack-helm post-run: playbooks/publish/post.yaml + vars: + base_version: "2025.2.0" - job: name: openstack-helm-deploy diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index b890601ac1..727557445a 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -18,6 +18,8 @@ # shared across all jobs helm_version: "3.18.1" chart_testing_version: "3.11.0" + virtualenv: "{{ ansible_user_dir }}/venv" + base_version: "2025.2.0" templates: - publish-openstack-docs-pti