From 202cce830e4e0d63684fefc74dfc8d6aaa9248c6 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Sat, 9 May 2020 11:49:42 -0500 Subject: [PATCH] Add new non-npm specific javascript jobs We have a bunch of jobs that are built around the npm role, but for projects using yarn, that can lead to ignoring yarn.lock. For projects with a yarn.lock, we can assume the user wants to use yarn. Make a new js-package-manager role that can detect if that's the case and otherwise use npm. Make an js_build_tool parameter that allows the user to override that auto-detection. Make a whole new suite of jobs that do this behavior, do not have npm in their name, and default to the latest node LTS, version 14. Don't install yarn if we're not going to use yarn. Also allow people who want to use yarn but don't have a yarn.lock to override js_build_tool everywhere we do that logic. Mark the old jobs deprecated. Shift the npm and yarn roles to use the new js-package-manager role with defaults set. Change-Id: I8013228ca05607a69f390a9bb75991fc6543f865 --- doc/source/deprecated-jobs.rst | 10 + doc/source/jobs.rst | 1 + doc/source/js-jobs.rst | 14 +- doc/source/js-roles.rst | 1 + playbooks/javascript/pre.yaml | 36 ++- playbooks/javascript/run.yaml | 2 +- playbooks/nodejs-npm/pre.yaml | 9 + playbooks/nodejs-npm/run.yaml | 4 + roles/ensure-javascript-packages/README.rst | 6 + .../tasks/main.yaml | 14 +- roles/js-package-manager/README.rst | 21 ++ roles/js-package-manager/defaults/main.yaml | 1 + roles/js-package-manager/tasks/main.yaml | 39 +++ .../vars/main.yaml | 0 roles/npm/tasks/main.yaml | 26 +- roles/yarn/tasks/main.yaml | 24 +- roles/yarn/vars/main.yaml | 9 - zuul.d/js-jobs.yaml | 224 +++++++++++++++++- 18 files changed, 374 insertions(+), 67 deletions(-) create mode 100644 doc/source/deprecated-jobs.rst create mode 100644 playbooks/nodejs-npm/pre.yaml create mode 100644 playbooks/nodejs-npm/run.yaml create mode 100644 roles/js-package-manager/README.rst create mode 100644 roles/js-package-manager/defaults/main.yaml create mode 100644 roles/js-package-manager/tasks/main.yaml rename roles/{npm => js-package-manager}/vars/main.yaml (100%) delete mode 100644 roles/yarn/vars/main.yaml diff --git a/doc/source/deprecated-jobs.rst b/doc/source/deprecated-jobs.rst new file mode 100644 index 000000000..a2ddd6088 --- /dev/null +++ b/doc/source/deprecated-jobs.rst @@ -0,0 +1,10 @@ +Deprecrated Jobs +================ + +.. zuul:autojob:: build-javascript-tarball +.. zuul:autojob:: build-javascript-content +.. zuul:autojob:: build-javascript-content-tarball +.. zuul:autojob:: nodejs-npm +.. zuul:autojob:: nodejs-npm-run-test +.. zuul:autojob:: nodejs-npm-run-lint +.. zuul:autojob:: nodejs-npm-run-docs diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index b6a131a43..3f61a29a4 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -13,3 +13,4 @@ Jobs helm-jobs packer-jobs system-jobs + deprecated-jobs diff --git a/doc/source/js-jobs.rst b/doc/source/js-jobs.rst index e78e84ab7..5ece2681c 100644 --- a/doc/source/js-jobs.rst +++ b/doc/source/js-jobs.rst @@ -1,10 +1,10 @@ Javascript Jobs =============== -.. zuul:autojob:: build-javascript-tarball -.. zuul:autojob:: build-javascript-content -.. zuul:autojob:: build-javascript-content-tarball -.. zuul:autojob:: nodejs-npm -.. zuul:autojob:: nodejs-npm-run-test -.. zuul:autojob:: nodejs-npm-run-lint -.. zuul:autojob:: nodejs-npm-run-docs +.. zuul:autojob:: build-javascript-source-tarball +.. zuul:autojob:: build-javascript-deployment +.. zuul:autojob:: build-javascript-deployment-tarball +.. zuul:autojob:: js-build +.. zuul:autojob:: nodejs-run-test +.. zuul:autojob:: nodejs-run-lint +.. zuul:autojob:: nodejs-run-docs diff --git a/doc/source/js-roles.rst b/doc/source/js-roles.rst index 2f502027c..449cd5a06 100644 --- a/doc/source/js-roles.rst +++ b/doc/source/js-roles.rst @@ -10,6 +10,7 @@ Javascript Roles .. zuul:autorole:: install-javascript-packages .. zuul:autorole:: install-nodejs .. zuul:autorole:: install-yarn +.. zuul:autorole:: js-package-manager .. zuul:autorole:: nodejs-test-dependencies .. zuul:autorole:: npm .. zuul:autorole:: upload-npm diff --git a/playbooks/javascript/pre.yaml b/playbooks/javascript/pre.yaml index b8c3ff5fa..c80609e42 100644 --- a/playbooks/javascript/pre.yaml +++ b/playbooks/javascript/pre.yaml @@ -1,4 +1,34 @@ - hosts: all - roles: - - ensure-yarn - - ensure-javascript-packages + tasks: + - name: Set node version if not set + set_fact: + node_version: '14' + when: node_version is not defined + + - name: Check for yarn.lock + when: js_build_tool is not defined + stat: + path: "{{ zuul_work_dir }}/yarn.lock" + get_checksum: false + get_mime: false + get_md5: false + register: yarn_lock_exists + + - name: Set js_build_tool fact + set_fact: + js_build_tool: '{{ yarn_lock_exists.stat.exists | ternary("yarn", "npm") }}' + when: js_build_tool is not defined + + - name: Ensure yarn if needed + include_role: + name: ensure-yarn + when: js_build_tool == 'yarn' + + - name: Ensure nodejs if needed + include_role: + name: ensure-nodejs + when: js_build_tool == 'npm' + + - name: Install javascript depends + include_role: + name: ensure-javascript-packages diff --git a/playbooks/javascript/run.yaml b/playbooks/javascript/run.yaml index c712dd0e5..b169e17b9 100644 --- a/playbooks/javascript/run.yaml +++ b/playbooks/javascript/run.yaml @@ -1,4 +1,4 @@ - hosts: all roles: - revoke-sudo - - npm + - js-package-manager diff --git a/playbooks/nodejs-npm/pre.yaml b/playbooks/nodejs-npm/pre.yaml new file mode 100644 index 000000000..79eb298d9 --- /dev/null +++ b/playbooks/nodejs-npm/pre.yaml @@ -0,0 +1,9 @@ +- hosts: all + pre_tasks: + - name: Set node version if not set + set_fact: + node_version: '6' + when: node_version is not defined + roles: + - ensure-yarn + - ensure-javascript-packages diff --git a/playbooks/nodejs-npm/run.yaml b/playbooks/nodejs-npm/run.yaml new file mode 100644 index 000000000..c712dd0e5 --- /dev/null +++ b/playbooks/nodejs-npm/run.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - revoke-sudo + - npm diff --git a/roles/ensure-javascript-packages/README.rst b/roles/ensure-javascript-packages/README.rst index 949dd14ac..1022b92b5 100644 --- a/roles/ensure-javascript-packages/README.rst +++ b/roles/ensure-javascript-packages/README.rst @@ -2,6 +2,12 @@ Install javascript dependencies needed for a project **Role Variables** +.. zuul:rolevar:: js_build_tool + :default: autodetected + + What command to use. If the ``zuul_work_dir`` has a ``yarn.lock`` + file it will default to ``yarn``, otherwise ``npm``. + .. zuul:rolevar:: zuul_work_dir :default: {{ zuul.project.src_dir }} diff --git a/roles/ensure-javascript-packages/tasks/main.yaml b/roles/ensure-javascript-packages/tasks/main.yaml index 70f3443f8..d96fe65eb 100644 --- a/roles/ensure-javascript-packages/tasks/main.yaml +++ b/roles/ensure-javascript-packages/tasks/main.yaml @@ -20,13 +20,19 @@ UPPER_CONSTRAINTS_FILE: "{{ tox_constraints_file }}" when: tox_constraints_file is defined -- name: Check for yarn.lock file +- name: Check for yarn.lock + when: js_build_tool is not defined stat: path: "{{ zuul_work_dir }}/yarn.lock" get_checksum: false get_mime: false get_md5: false - register: yarn_lock + register: yarn_lock_exists + +- name: Set js_build_tool fact + set_fact: + js_build_tool: '{{ yarn_lock_exists.stat.exists | ternary("yarn", "npm") }}' + when: js_build_tool is not defined - name: Install yarn dependencies command: yarn install @@ -34,11 +40,11 @@ DISPLAY: ':99' args: chdir: "{{ zuul_work_dir }}" - when: yarn_lock.stat.exists + when: js_build_tool == 'yarn' - name: Install npm dependencies command: npm install --verbose environment: "{{ npm_environment|combine(tox_constraints_env|default({})) }}" args: chdir: "{{ zuul_work_dir }}" - when: not yarn_lock.stat.exists + when: js_build_tool == 'npm' diff --git a/roles/js-package-manager/README.rst b/roles/js-package-manager/README.rst new file mode 100644 index 000000000..86a12b7a1 --- /dev/null +++ b/roles/js-package-manager/README.rst @@ -0,0 +1,21 @@ +Run javascript build command in a source directory. Assumes the appropriate version +of npm or yarn has been installed. + +**Role Variables** + +.. zuul:rolevar:: js_build_tool + :default: autodetected + + What command to use. If the ``zuul_work_dir`` has a ``yarn.lock`` + file it will default to ``yarn``, otherwise ``npm``. + +.. zuul:rolevar:: js_build_command + + Command to run. If it's a standard lifecycle command, it will be run as + ``{{ js_build_tool }} {{ js_build_command }}``. Otherwise it will be run as + ``{{ js_build_tool }} run {{ js_build_command }}``. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to run in. diff --git a/roles/js-package-manager/defaults/main.yaml b/roles/js-package-manager/defaults/main.yaml new file mode 100644 index 000000000..9739eb171 --- /dev/null +++ b/roles/js-package-manager/defaults/main.yaml @@ -0,0 +1 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" diff --git a/roles/js-package-manager/tasks/main.yaml b/roles/js-package-manager/tasks/main.yaml new file mode 100644 index 000000000..65e82b722 --- /dev/null +++ b/roles/js-package-manager/tasks/main.yaml @@ -0,0 +1,39 @@ +- name: Require js_build_command variable + fail: + msg: js_build_command is required for this role + when: + - js_build_command is not defined + - npm_command is not defined + +# Set this unconditionally because people who were using old +# versions of things with npm_command to override don't want to +# stop overriding. We can remove this once we remove the npm +# versions of the jobs. +- name: Set js_build_command to npm_command for backwards compat + when: npm_command is defined + set_fact: + js_build_command: '{{ npm_command }}' + +- name: Check for yarn.lock + when: js_build_tool is not defined + stat: + path: "{{ zuul_work_dir }}/yarn.lock" + get_checksum: false + get_mime: false + get_md5: false + register: yarn_lock_exists + +- name: Set js_build_tool fact + set_fact: + js_build_tool: '{{ yarn_lock_exists.stat.exists | ternary("yarn", "npm") }}' + when: js_build_tool is not defined + +- name: Run js build command + command: "{{ js_build_tool }} {% if js_build_command not in npm_lifecycle_phases %} run {% endif %} {{ js_build_command }} --verbose" + # Need to set DISPLAY to the value that will be set when the virtual + # framebuffer is set up for doing browser tests. + environment: + DISPLAY: ':99' + CI: 'true' + args: + chdir: "{{ zuul_work_dir }}" diff --git a/roles/npm/vars/main.yaml b/roles/js-package-manager/vars/main.yaml similarity index 100% rename from roles/npm/vars/main.yaml rename to roles/js-package-manager/vars/main.yaml diff --git a/roles/npm/tasks/main.yaml b/roles/npm/tasks/main.yaml index c3bd67f52..91588145b 100644 --- a/roles/npm/tasks/main.yaml +++ b/roles/npm/tasks/main.yaml @@ -3,24 +3,8 @@ msg: npm_command is required for this role when: npm_command is not defined -- name: Run npm lifecycle command - when: npm_command in npm_lifecycle_phases - command: "npm {{ npm_command }} --verbose" - # Need to set DISPLAY to the value that will be set when the virtual - # framebuffer is set up for doing browser tests. - environment: - DISPLAY: ':99' - CI: 'true' - args: - chdir: "{{ zuul_work_dir }}" - -- name: Run npm custom command - when: npm_command not in npm_lifecycle_phases - command: "npm run {{ npm_command }} --verbose" - # Need to set DISPLAY to the value that will be set when the virtual - # framebuffer is set up for doing browser tests. - environment: - DISPLAY: ':99' - CI: 'true' - args: - chdir: "{{ zuul_work_dir }}" +- include_role: + name: js-package-manager + vars: + js_build_command: '{{ npm_command }}' + js_build_tool: 'npm' diff --git a/roles/yarn/tasks/main.yaml b/roles/yarn/tasks/main.yaml index e51187c68..16fb0444e 100644 --- a/roles/yarn/tasks/main.yaml +++ b/roles/yarn/tasks/main.yaml @@ -3,22 +3,8 @@ msg: yarn_command is required for this role when: yarn_command is not defined -- name: Run yarn lifecycle command - when: yarn_command in yarn_lifecycle_phases - command: "yarn {{ yarn_command }}" - # Need to set DISPLAY to the value that will be set when the virtual - # framebuffer is set up for doing browser tests. - environment: - DISPLAY: ':99' - args: - chdir: "{{ zuul_work_dir }}" - -- name: Run yarn custom command - when: yarn_command not in yarn_lifecycle_phases - command: "yarn run {{ yarn_command }}" - # Need to set DISPLAY to the value that will be set when the virtual - # framebuffer is set up for doing browser tests. - environment: - DISPLAY: ':99' - args: - chdir: "{{ zuul_work_dir }}" +- include_role: + name: js-package-manager + vars: + js_build_command: '{{ yarn_command }}' + js_build_tool: 'yarn' diff --git a/roles/yarn/vars/main.yaml b/roles/yarn/vars/main.yaml deleted file mode 100644 index 5343f7036..000000000 --- a/roles/yarn/vars/main.yaml +++ /dev/null @@ -1,9 +0,0 @@ -yarn_lifecycle_phases: - - install - - pack - - publish - - restart - - start - - stop - - test - - version diff --git a/zuul.d/js-jobs.yaml b/zuul.d/js-jobs.yaml index 1c2047660..ac20431df 100644 --- a/zuul.d/js-jobs.yaml +++ b/zuul.d/js-jobs.yaml @@ -1,10 +1,211 @@ # Jobs listed in js-jobs.rst. +- job: + name: js-build + parent: unittests + description: | + Base job for javascript operations + + Responds to these variables: + + .. zuul:jobvar:: js_build_command + :default: build + + Command to pass to the javascript package manager.. + + .. zuul:jobvar:: js_build_tool + :default: autodetected + + Command to use for running the package manager, such as npm or yarn. + + .. zuul:jobvar:: node_version + :default: 14 + + The version of Node to use. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Path to operate in. + + .. zuul:jobvar:: javascript_content_dir + :default: dist + + Directory, relative to zuul_work_dir, holding build content. + pre-run: playbooks/javascript/pre.yaml + run: playbooks/javascript/run.yaml + post-run: playbooks/javascript/post.yaml + vars: + js_build_command: build + +- job: + name: build-javascript-source-tarball + parent: js-build + description: | + Build a source tarball for a Javascript project + + Responds to these variables: + + .. zuul:jobvar:: node_version + :default: 14 + + The version of Node to use. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Path to operate in. + + .. zuul:jobvar:: javascript_content_dir + :default: dist + + Directory, relative to zuul_work_dir, holding build content. + vars: + js_build_command: pack + +- job: + name: build-javascript-deployment + parent: js-build + description: | + Build javascript web content as it should be deployed. + + Responds to these variables: + + .. zuul:jobvar:: js_build_command + :default: build + + Command to pass to npm. + + .. zuul:jobvar:: node_version + :default: 14 + + The version of Node to use. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Path to operate in. + + .. zuul:jobvar:: javascript_content_dir + :default: dist + + Directory, relative to zuul_work_dir, holding build content. + success-url: npm/html/ + +- job: + name: build-javascript-deployment-tarball + parent: js-build + description: | + Build an archive of javascript web content as it should be deployed. + + Responds to these variables: + + .. zuul:jobvar:: js_build_command + :default: build + + Command to pass to npm. + + .. zuul:jobvar:: node_version + :default: 14 + + The version of Node to use. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Path to operate in. + + .. zuul:jobvar:: javascript_content_dir + :default: dist + + Directory, relative to zuul_work_dir, holding build content. + + .. zuul:jobvar:: create_tarball_directory + + Create a tarball with the contents of + create_tarball_directory (relative to zuul_work_dir). + post-run: playbooks/javascript/tarball.yaml + +- job: + name: nodejs-run-test + parent: js-build + description: | + Run test using nodejs. This test also starts Xvfb for run time + tests. + + Responds to these variables: + + .. zuul:jobvar:: node_version + :default: 14 + + The version of Node to use. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Path to operate in. + + .. zuul:jobvar:: javascript_content_dir + :default: dist + + Directory, relative to zuul_work_dir, holding build content. + pre-run: playbooks/javascript/pre-test.yaml + vars: + js_build_command: test + +- job: + name: nodejs-run-lint + parent: js-build + description: | + Run lint using nodejs. + + Responds to these variables: + + .. zuul:jobvar:: node_version + :default: 14 + + The version of Node to use. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Path to operate in. + + .. zuul:jobvar:: javascript_content_dir + :default: dist + + Directory, relative to zuul_work_dir, holding build content. + vars: + js_build_command: lint + +- job: + name: nodejs-run-docs + parent: js-build + description: | + Run docs using nodejs. + + Responds to these variables: + + .. zuul:jobvar:: node_version + :default: 14 + + The version of Node to use. + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Path to operate in. + post-run: playbooks/tox/docs-post.yaml + success-url: docs/ + vars: + js_build_command: docs + - job: name: nodejs-npm parent: unittests description: | - Base job for javascript operations + Base job for javascript operations using npm. + ** DEPRECATED: Switch to js-build ** Responds to these variables: @@ -27,8 +228,8 @@ :default: dist Directory, relative to zuul_work_dir, holding build content. - pre-run: playbooks/javascript/pre.yaml - run: playbooks/javascript/run.yaml + pre-run: playbooks/nodejs-npm/pre.yaml + run: playbooks/nodejs-npm/run.yaml post-run: playbooks/javascript/post.yaml vars: npm_command: build @@ -39,6 +240,8 @@ description: | Build a source tarball for a Javascript project + ** DEPRECATED: Switch to build-javascript-source-tarball ** + Responds to these variables: .. zuul:jobvar:: node_version @@ -64,6 +267,8 @@ description: | Build javascript web content as it should be deployed. + ** DEPRECATED: Please switch to build-javascript-release ** + Responds to these variables: .. zuul:jobvar:: npm_command @@ -93,6 +298,8 @@ description: | Build an archive of javascript web content as it should be deployed. + ** DEPRECATED: Please use build-javascript-deployment-tarball ** + Responds to these variables: .. zuul:jobvar:: npm_command @@ -128,6 +335,8 @@ Run test using nodejs. This test also starts Xvfb for run time tests. + ** DEPRECATED: Please switch to nodejs-run-test ** + Responds to these variables: .. zuul:jobvar:: node_version @@ -154,6 +363,8 @@ description: | Run lint using nodejs. + ** DEPRECATED: Please switch to nodejs-run-lint ** + Responds to these variables: .. zuul:jobvar:: node_version @@ -179,8 +390,15 @@ description: | Run docs using nodejs. + ** DEPRECATED: Please switch to nodejs-run-docs ** + Responds to these variables: + .. zuul:jobvar:: npm_executable + :default: autodetected + + Command to use for running npm, such as npm or yarn. + .. zuul:jobvar:: node_version :default: 6