diff --git a/nodepool/elements/puppet/bin/prepare-node b/nodepool/elements/puppet/bin/prepare-node
index 941265176f..37c90ac240 100644
--- a/nodepool/elements/puppet/bin/prepare-node
+++ b/nodepool/elements/puppet/bin/prepare-node
@@ -58,3 +58,5 @@ set -e
 
 # Make sure resolv.conf settings don't break dib
 echo "nameserver $NODEPOOL_STATIC_NAMESERVER"> /etc/resolv.conf
+
+rm -f /etc/ssh/ssh_host_*
diff --git a/nodepool/scripts/prepare_node.sh b/nodepool/scripts/prepare_node.sh
index 9b19331855..9eea6f97c1 100755
--- a/nodepool/scripts/prepare_node.sh
+++ b/nodepool/scripts/prepare_node.sh
@@ -173,5 +173,7 @@ sudo -H virtualenv /usr/zuul-swift-logs-env
 sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \
     requests glob2
 
+sudo rm -f /etc/ssh/ssh_host_*
+
 sync
 sleep 5