diff --git a/.zuul.yaml b/.zuul.yaml
index 7ca15ba68..03774fc28 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -344,14 +344,6 @@
     vars:
       puppet: 4
 
-- job:
-    name: puppet-openstack-module-build
-    parent: puppet-openstack-module-base
-    run: playbooks/run-build-tests.yaml
-    files: ^metadata.json$
-    vars:
-      use_puppetlabs: true
-
 - job:
     name: puppet-openstack-unit-base
     parent: puppet-openstack-integration-base
@@ -417,16 +409,18 @@
     name: puppet-openstack-check-jobs
     check:
       jobs:
+        - build-openstack-puppet-tarball:
+            files: ^metadata.json$
         - puppet-openstack-lint
         - puppet-openstack-syntax-3-legacy
         - puppet-openstack-syntax-4
-        - puppet-openstack-module-build
     gate:
       jobs:
+        - build-openstack-puppet-tarball:
+            files: ^metadata.json$
         - puppet-openstack-lint
         - puppet-openstack-syntax-3-legacy
         - puppet-openstack-syntax-4
-        - puppet-openstack-module-build
 
 - project:
     name: openstack/puppet-openstack-integration
diff --git a/playbooks/run-build-tests.yaml b/playbooks/run-build-tests.yaml
deleted file mode 100644
index 092662e6f..000000000
--- a/playbooks/run-build-tests.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-- hosts: all
-  tasks:
-    - shell: /opt/puppetlabs/bin/puppet module build .
-      args:
-        chdir: '{{ zuul.project.src_dir }}'