From 98d6d62c3a7bf8c66bba70183fe2164381769799 Mon Sep 17 00:00:00 2001 From: Dmitry Tantsur Date: Tue, 27 Aug 2019 17:29:51 +0200 Subject: [PATCH] Import ansible version of the image building job This patch starts "ansiblization" of the image building script, while also cleaning it up. More importantly, it adds check and gate jobs that verify the same playbooks as used in the post job. Also correct source-repository for ironic-python-agent to use the project name, not the old element name. Change-Id: I9a404b24f95e47c84b22d4739fcf2b5b800886f9 --- .zuul.yaml | 68 +++++++++++++++++-- .../60-ironic-agent-install | 2 +- .../source-repository-ironic-agent | 1 - .../source-repository-ironic-python-agent | 1 + .../ironic-python-agent-build-image/post.yaml | 47 +++++++++++++ .../ironic-python-agent-build-image/pre.yaml | 7 ++ .../ironic-python-agent-build-image/run.yaml | 25 +++++++ roles/ipa-build-dib-image/defaults/main.yaml | 5 ++ roles/ipa-build-dib-image/tasks/install.yaml | 16 +++++ roles/ipa-build-dib-image/tasks/main.yaml | 16 +++++ .../defaults/main.yaml | 4 ++ roles/ipa-build-tinyipa-image/tasks/main.yaml | 23 +++++++ 12 files changed, 206 insertions(+), 9 deletions(-) delete mode 100644 dib/ironic-python-agent-ramdisk/source-repository-ironic-agent create mode 100644 dib/ironic-python-agent-ramdisk/source-repository-ironic-python-agent create mode 100644 playbooks/ironic-python-agent-build-image/post.yaml create mode 100644 playbooks/ironic-python-agent-build-image/pre.yaml create mode 100644 playbooks/ironic-python-agent-build-image/run.yaml create mode 100644 roles/ipa-build-dib-image/defaults/main.yaml create mode 100644 roles/ipa-build-dib-image/tasks/install.yaml create mode 100644 roles/ipa-build-dib-image/tasks/main.yaml create mode 100644 roles/ipa-build-tinyipa-image/defaults/main.yaml create mode 100644 roles/ipa-build-tinyipa-image/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index 87aecba..b025570 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,3 +1,58 @@ +- job: + name: ironic-python-agent-build-image-base + parent: publish-openstack-artifacts + pre-run: playbooks/ironic-python-agent-build-image/pre.yaml + run: playbooks/ironic-python-agent-build-image/run.yaml + post-run: playbooks/ironic-python-build-image/post.yaml + timeout: 1800 + required-projects: + - openstack/requirements + - openstack/ironic-python-agent + - openstack/ironic-python-agent-builder + +- job: + name: ironic-python-agent-build-image-tinyipa + parent: ironic-python-agent-build-image-base + vars: + image_type: 'tinyipa' + +- job: + name: ironic-python-agent-build-image-dib-centos7 + parent: ironic-python-agent-build-image-base + required-projects: + # NOTE(dtantsur): used for bindep only + - openstack/diskimage-builder + vars: + image_type: 'dib' + image_distro: 'centos7' + +- job: + name: ironic-python-agent-check-image-base + parent: base + pre-run: playbooks/ironic-python-agent-build-image/pre.yaml + run: playbooks/ironic-python-agent-build-image/run.yaml + timeout: 1800 + required-projects: + - openstack/requirements + - openstack/ironic-python-agent + - openstack/ironic-python-agent-builder + +- job: + name: ironic-python-agent-check-image-tinyipa + parent: ironic-python-agent-check-image-base + vars: + image_type: 'tinyipa' + +- job: + name: ironic-python-agent-check-image-dib-centos7 + parent: ironic-python-agent-check-image-base + required-projects: + # NOTE(dtantsur): used for bindep only + - openstack/diskimage-builder + vars: + image_type: 'dib' + image_distro: 'centos7' + - project: templates: - publish-openstack-docs-pti @@ -11,6 +66,8 @@ - ipa-tempest-wholedisk-bios-ipmi-direct-tinyipa-src - ipa-tempest-wholedisk-bios-ipmi-iscsi-tinyipa-src - ipa-tempest-ironic-inspector-src + - ironic-python-agent-check-image-tinyipa + - ironic-python-agent-check-image-dib-centos7 gate: jobs: - openstack-tox-pep8 @@ -19,13 +76,10 @@ - ipa-tempest-wholedisk-bios-ipmi-direct-tinyipa-src - ipa-tempest-wholedisk-bios-ipmi-iscsi-tinyipa-src - ipa-tempest-ironic-inspector-src + - ironic-python-agent-check-image-tinyipa + - ironic-python-agent-check-image-dib-centos7 post: jobs: - publish-openstack-python-branch-tarball - - ipa-builder-buildimage-tinyipa - -- job: - name: ipa-builder-buildimage-tinyipa - parent: ironic-python-agent-buildimage-base - vars: - image_name: 'tinyipa' + - ironic-python-agent-build-image-tinyipa + - ironic-python-agent-build-image-dib-centos7 diff --git a/dib/ironic-python-agent-ramdisk/install.d/ironic-agent-source-install/60-ironic-agent-install b/dib/ironic-python-agent-ramdisk/install.d/ironic-agent-source-install/60-ironic-agent-install index 3987c14..e1496da 100755 --- a/dib/ironic-python-agent-ramdisk/install.d/ironic-agent-source-install/60-ironic-agent-install +++ b/dib/ironic-python-agent-ramdisk/install.d/ironic-agent-source-install/60-ironic-agent-install @@ -7,7 +7,7 @@ set -eu set -o pipefail SCRIPTDIR=$(dirname $0) -IPADIR=/usr/share/ironic-python-agent +IPADIR=/tmp/ironic-python-agent # Generate upper-constraints $IPADIR/imagebuild/common/generate_upper_constraints.sh $IPADIR/upper-constraints.txt diff --git a/dib/ironic-python-agent-ramdisk/source-repository-ironic-agent b/dib/ironic-python-agent-ramdisk/source-repository-ironic-agent deleted file mode 100644 index 72f690d..0000000 --- a/dib/ironic-python-agent-ramdisk/source-repository-ironic-agent +++ /dev/null @@ -1 +0,0 @@ -ironic-agent git /usr/share/ironic-python-agent https://opendev.org/openstack/ironic-python-agent diff --git a/dib/ironic-python-agent-ramdisk/source-repository-ironic-python-agent b/dib/ironic-python-agent-ramdisk/source-repository-ironic-python-agent new file mode 100644 index 0000000..9037311 --- /dev/null +++ b/dib/ironic-python-agent-ramdisk/source-repository-ironic-python-agent @@ -0,0 +1 @@ +ironic-python-agent git /tmp/ironic-python-agent https://opendev.org/openstack/ironic-python-agent diff --git a/playbooks/ironic-python-agent-build-image/post.yaml b/playbooks/ironic-python-agent-build-image/post.yaml new file mode 100644 index 0000000..f83668c --- /dev/null +++ b/playbooks/ironic-python-agent-build-image/post.yaml @@ -0,0 +1,47 @@ +- hosts: all + + vars: + ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW' + ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR' + + tasks: + + - name: Ensure artifacts directory exists + file: + path: '{{ zuul.executor.work_root }}/artifacts/{{ image_name }}/files' + state: directory + delegate_to: localhost + + - name: Copy files from {{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent on node + synchronize: + src: '{{ ipa_raw_dir }}' + dest: '{{ zuul.executor.work_root }}/artifacts/{{ image_name }}/files/' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent on node + synchronize: + src: '{{ ipa_tar_dir }}' + dest: '{{ zuul.executor.work_root }}/artifacts/{{ image_name }}/' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=/* + - --prune-empty-dirs + + - name: Copy files from {{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent on node + synchronize: + src: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/' + dest: '{{ zuul.executor.log_root }}' + mode: pull + copy_links: true + verify_host: true + rsync_opts: + - --include=*/ + - --exclude=* + - --prune-empty-dirs diff --git a/playbooks/ironic-python-agent-build-image/pre.yaml b/playbooks/ironic-python-agent-build-image/pre.yaml new file mode 100644 index 0000000..0ff909b --- /dev/null +++ b/playbooks/ironic-python-agent-build-image/pre.yaml @@ -0,0 +1,7 @@ +- hosts: all + tasks: + + - import_role: + name: ipa-build-dib-image + tasks_from: install.yaml + when: image_type == 'dib' diff --git a/playbooks/ironic-python-agent-build-image/run.yaml b/playbooks/ironic-python-agent-build-image/run.yaml new file mode 100644 index 0000000..bd060df --- /dev/null +++ b/playbooks/ironic-python-agent-build-image/run.yaml @@ -0,0 +1,25 @@ +- hosts: all + + vars: + ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW' + ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR' + + tasks: + + - name: Create a target directory for raw images + file: + path: '{{ ipa_raw_dir }}' + state: directory + + - name: Create a target directory for tarballs + file: + path: '{{ ipa_tar_dir }}' + state: directory + + - import_role: + name: ipa-build-tinyipa-image + when: image_type == 'tinyipa' + + - import_role: + name: ipa-build-dib-image + when: image_type == 'dib' diff --git a/roles/ipa-build-dib-image/defaults/main.yaml b/roles/ipa-build-dib-image/defaults/main.yaml new file mode 100644 index 0000000..f9ba01f --- /dev/null +++ b/roles/ipa-build-dib-image/defaults/main.yaml @@ -0,0 +1,5 @@ +image_distro: centos7 +ipa_branch_path: '{{ zuul.branch | replace("/", "-") }}' +ipa_source_path: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent' +ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW' +ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR' diff --git a/roles/ipa-build-dib-image/tasks/install.yaml b/roles/ipa-build-dib-image/tasks/install.yaml new file mode 100644 index 0000000..fc90b0a --- /dev/null +++ b/roles/ipa-build-dib-image/tasks/install.yaml @@ -0,0 +1,16 @@ +- name: Install binary dependencies from diskimage-builder + include_role: + name: bindep + vars: + bindep_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/diskimage-builder'].src_dir }}" + +- name: Install requirements for ironic-python-agent-builder + pip: + requirements: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/ironic-python-agent-builder'].src_dir }}/requirements.txt" + extra_args: -c "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt" + become: true + +- name: Install ironic-python-agent-builder + pip: + name: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/ironic-python-agent-builder'].src_dir }}" + become: true diff --git a/roles/ipa-build-dib-image/tasks/main.yaml b/roles/ipa-build-dib-image/tasks/main.yaml new file mode 100644 index 0000000..71a4b0e --- /dev/null +++ b/roles/ipa-build-dib-image/tasks/main.yaml @@ -0,0 +1,16 @@ +- name: Generate image name + set_fact: + image_name: ipa-{{ image_distro }}-{{ ipa_branch_path }} + +- name: Build a DIB image + command: | + ironic-python-agent-builder -e dhcp-all-interfaces -o {{ image_name }} {{ image_distro }} + environment: + # Use repositories checked out by Zuul + DIB_REPOLOCATION_ironic_python_agent: '{{ ipa_source_path }}' + DIB_REPOREF_ironic_python_agent: HEAD + +- name: Move the resulting files + shell: | + tar -czf "{{ ipa_tar_dir }}/{{ image_name }}.tar.gz" {{ image_name }}* + mv {{ image_name }}* "{{ ipa_raw_dir }}" diff --git a/roles/ipa-build-tinyipa-image/defaults/main.yaml b/roles/ipa-build-tinyipa-image/defaults/main.yaml new file mode 100644 index 0000000..0f62135 --- /dev/null +++ b/roles/ipa-build-tinyipa-image/defaults/main.yaml @@ -0,0 +1,4 @@ +ipa_branch_path: '{{ zuul.branch | replace("/", "-") }}' +ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW' +ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR' +tinyipa_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent-builder/tinyipa' diff --git a/roles/ipa-build-tinyipa-image/tasks/main.yaml b/roles/ipa-build-tinyipa-image/tasks/main.yaml new file mode 100644 index 0000000..d9acb63 --- /dev/null +++ b/roles/ipa-build-tinyipa-image/tasks/main.yaml @@ -0,0 +1,23 @@ +- name: Build a TinyIPA image + command: make + args: + chdir: '{{ tinyipa_dir }}' + environment: + BRANCH_PATH: '{{ ipa_branch_path }}' + BUILD_AND_INSTALL_TINYIPA: True + IPA_SOURCE_DIR: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent' + +- name: Move resulting files + shell: | + mv tinyipa*.tar.gz* "{{ ipa_tar_dir }}" + mv tinyipa*.* "{{ ipa_raw_dir }}" + mv tiny-instance-uec*.tar.gz "{{ ipa_tar_dir }}" + args: + chdir: '{{ tinyipa_dir }}' + +# NOTE(sambetts) Must clean up chroot directories before +# publisher rsync command tries and fails to read them +- name: Clean up the environment + command: make clean + args: + chdir: '{{ tinyipa_dir }}'