openstack
/
openstack-doc-tools
Code Issues Proposed changes

[sitemap] respect the robots.txt policies 

Change-Id: I219eff6241479c3c883c55079b48a88591a2a99d
This commit is contained in:
Christian Berendt 2015-10-16 07:59:33 +02:00
parent 512d64fd78
commit 8f05ac1b0a
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sitemap/generator/settings.py
View File

@ -31,5 +31,6 @@ DOWNLOAD_WARNSIZE = 67108864
LOG_LEVEL = 'INFO' LOG_LEVEL = 'INFO'
LOGGING_ENABLED = True LOGGING_ENABLED = True
RANDOMIZE_DOWNLOAD_DELAY = False RANDOMIZE_DOWNLOAD_DELAY = False
ROBOTSTXT_OBEY = True
TELNETCONSOLE_ENABLED = False TELNETCONSOLE_ENABLED = False
linkextractors.IGNORED_EXTENSIONS.remove('pdf') linkextractors.IGNORED_EXTENSIONS.remove('pdf')