[sitemap] respect the robots.txt policies
Change-Id: I219eff6241479c3c883c55079b48a88591a2a99d
This commit is contained in:
parent
512d64fd78
commit
8f05ac1b0a
@ -31,5 +31,6 @@ DOWNLOAD_WARNSIZE = 67108864
|
|||||||
LOG_LEVEL = 'INFO'
|
LOG_LEVEL = 'INFO'
|
||||||
LOGGING_ENABLED = True
|
LOGGING_ENABLED = True
|
||||||
RANDOMIZE_DOWNLOAD_DELAY = False
|
RANDOMIZE_DOWNLOAD_DELAY = False
|
||||||
|
ROBOTSTXT_OBEY = True
|
||||||
TELNETCONSOLE_ENABLED = False
|
TELNETCONSOLE_ENABLED = False
|
||||||
linkextractors.IGNORED_EXTENSIONS.remove('pdf')
|
linkextractors.IGNORED_EXTENSIONS.remove('pdf')
|
||||||
|
Loading…
x
Reference in New Issue
Block a user