diff --git a/galaxy.yml.in b/galaxy.yml.in
index fb0d692b..2852eb46 100644
--- a/galaxy.yml.in
+++ b/galaxy.yml.in
@@ -4,7 +4,6 @@ readme: README.md
 authors: Openstack
 description: Openstack Ansible modules
 license: GPL-3.0-or-later
-license_file: COPYING
 tags:
   - cloud
   - openstack
diff --git a/test-requirements.txt b/test-requirements.txt
index 9e29b4b3..2f45aec3 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -8,3 +8,4 @@ voluptuous
 yamllint
 rstcheck
 ruamel.yaml
+galaxy-importer
diff --git a/tools/check-import.sh b/tools/check-import.sh
new file mode 100755
index 00000000..af4caf5c
--- /dev/null
+++ b/tools/check-import.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+TOXDIR=${1:-.}
+
+# galaxy_importer.main does not return non-zero error code on error
+output=$(python -m galaxy_importer.main $TOXDIR/build_artifact/*)
+if echo $output | grep ERROR: ; then
+    exit 1
+fi
diff --git a/tox.ini b/tox.ini
index bb21394d..af3c94ae 100644
--- a/tox.ini
+++ b/tox.ini
@@ -34,10 +34,12 @@ commands =
 deps =
     pbr
     ruamel.yaml
+    galaxy-importer
     git+https://github.com/ansible/ansible
 commands =
     python {toxinidir}/tools/build.py
     ansible-galaxy collection build --force {toxinidir} --output-path {toxinidir}/build_artifact
+    /bin/bash {toxinidir}/tools/check-import.sh {toxinidir}
 
 [testenv:linters]
 passenv = *