diff --git a/sitemap/generator/settings.py b/sitemap/generator/settings.py
index 4bab6681..6f2b2cb1 100644
--- a/sitemap/generator/settings.py
+++ b/sitemap/generator/settings.py
@@ -31,5 +31,6 @@ DOWNLOAD_WARNSIZE = 67108864
 LOG_LEVEL = 'INFO'
 LOGGING_ENABLED = True
 RANDOMIZE_DOWNLOAD_DELAY = False
+ROBOTSTXT_OBEY = True
 TELNETCONSOLE_ENABLED = False
 linkextractors.IGNORED_EXTENSIONS.remove('pdf')