Merge "[sitemap] respect the robots.txt policies"

changes/48/237848/1
Jenkins 7 years ago committed by Gerrit Code Review
commit c47df24295
  1. 1
      sitemap/generator/settings.py

@ -28,5 +28,6 @@ DOWNLOAD_WARNSIZE = 67108864
LOG_LEVEL = 'INFO'
LOGGING_ENABLED = True
RANDOMIZE_DOWNLOAD_DELAY = False
ROBOTSTXT_OBEY = True
TELNETCONSOLE_ENABLED = False
linkextractors.IGNORED_EXTENSIONS.remove('pdf')

Loading…
Cancel
Save