diff --git a/sitemap/generator/settings.py b/sitemap/generator/settings.py
index 1f4f76ec..eba55850 100644
--- a/sitemap/generator/settings.py
+++ b/sitemap/generator/settings.py
@@ -28,5 +28,6 @@ DOWNLOAD_WARNSIZE = 67108864
 LOG_LEVEL = 'INFO'
 LOGGING_ENABLED = True
 RANDOMIZE_DOWNLOAD_DELAY = False
+ROBOTSTXT_OBEY = True
 TELNETCONSOLE_ENABLED = False
 linkextractors.IGNORED_EXTENSIONS.remove('pdf')