From 8f05ac1b0ad01af0c13f284de872f74913371407 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Fri, 16 Oct 2015 07:59:33 +0200 Subject: [PATCH] [sitemap] respect the robots.txt policies Change-Id: I219eff6241479c3c883c55079b48a88591a2a99d --- sitemap/generator/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sitemap/generator/settings.py b/sitemap/generator/settings.py index 4bab6681..6f2b2cb1 100644 --- a/sitemap/generator/settings.py +++ b/sitemap/generator/settings.py @@ -31,5 +31,6 @@ DOWNLOAD_WARNSIZE = 67108864 LOG_LEVEL = 'INFO' LOGGING_ENABLED = True RANDOMIZE_DOWNLOAD_DELAY = False +ROBOTSTXT_OBEY = True TELNETCONSOLE_ENABLED = False linkextractors.IGNORED_EXTENSIONS.remove('pdf')