openstack/openstack-doc-tools
Code Issues Proposed changes

Merge "[sitemap] respect the robots.txt policies"

Browse Source
This commit is contained in:
Jenkins
2015-10-20 12:15:50 +00:00
committed by Gerrit Code Review
parent 9c5b3e7171 8f05ac1b0a
commit c47df24295
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
sitemap/generator/settings.py
View File

@@ -28,5 +28,6 @@ DOWNLOAD_WARNSIZE = 67108864
LOG_LEVEL = 'INFO'
LOGGING_ENABLED = True
RANDOMIZE_DOWNLOAD_DELAY = False
ROBOTSTXT_OBEY = True
TELNETCONSOLE_ENABLED = False
linkextractors.IGNORED_EXTENSIONS.remove('pdf')