Merge "[sitemap] respect the robots.txt policies"
This commit is contained in:
commit
c47df24295
|
@ -28,5 +28,6 @@ DOWNLOAD_WARNSIZE = 67108864
|
||||||
LOG_LEVEL = 'INFO'
|
LOG_LEVEL = 'INFO'
|
||||||
LOGGING_ENABLED = True
|
LOGGING_ENABLED = True
|
||||||
RANDOMIZE_DOWNLOAD_DELAY = False
|
RANDOMIZE_DOWNLOAD_DELAY = False
|
||||||
|
ROBOTSTXT_OBEY = True
|
||||||
TELNETCONSOLE_ENABLED = False
|
TELNETCONSOLE_ENABLED = False
|
||||||
linkextractors.IGNORED_EXTENSIONS.remove('pdf')
|
linkextractors.IGNORED_EXTENSIONS.remove('pdf')
|
||||||
|
|
Loading…
Reference in New Issue