Browse Source

Merge "[sitemap] respect the robots.txt policies"

tags/0.32.0
Jenkins 3 years ago
parent
commit
c47df24295
1 changed files with 1 additions and 0 deletions
  1. 1
    0
      sitemap/generator/settings.py

+ 1
- 0
sitemap/generator/settings.py View File

@@ -28,5 +28,6 @@ DOWNLOAD_WARNSIZE = 67108864
28 28
 LOG_LEVEL = 'INFO'
29 29
 LOGGING_ENABLED = True
30 30
 RANDOMIZE_DOWNLOAD_DELAY = False
31
+ROBOTSTXT_OBEY = True
31 32
 TELNETCONSOLE_ENABLED = False
32 33
 linkextractors.IGNORED_EXTENSIONS.remove('pdf')

Loading…
Cancel
Save