From 8f05ac1b0ad01af0c13f284de872f74913371407 Mon Sep 17 00:00:00 2001
From: Christian Berendt <berendt@b1-systems.de>
Date: Fri, 16 Oct 2015 07:59:33 +0200
Subject: [PATCH] [sitemap] respect the robots.txt policies

Change-Id: I219eff6241479c3c883c55079b48a88591a2a99d
---
 sitemap/generator/settings.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sitemap/generator/settings.py b/sitemap/generator/settings.py
index 4bab6681..6f2b2cb1 100644
--- a/sitemap/generator/settings.py
+++ b/sitemap/generator/settings.py
@@ -31,5 +31,6 @@ DOWNLOAD_WARNSIZE = 67108864
 LOG_LEVEL = 'INFO'
 LOGGING_ENABLED = True
 RANDOMIZE_DOWNLOAD_DELAY = False
+ROBOTSTXT_OBEY = True
 TELNETCONSOLE_ENABLED = False
 linkextractors.IGNORED_EXTENSIONS.remove('pdf')