[sitemap] if last-modified not set in header use current date
This solves the following issue when last-modified is not set in the response header: ---snip--- KeyError: 'Last-Modified' ---snap--- Change-Id: I372f9638f6b5006ff250b9dbe0e75cc1556d18c2
This commit is contained in:
parent
512d64fd78
commit
6b5b7d530a
|
@ -72,7 +72,10 @@ class SitemapSpider(spiders.CrawlSpider):
|
|||
if path.startswith("/%s" % entry):
|
||||
item['changefreq'] = 'weekly'
|
||||
|
||||
lastmod = time.strptime(response.headers['Last-Modified'],
|
||||
"%a, %d %b %Y %H:%M:%S %Z")
|
||||
if 'Last-Modified' in response.headers:
|
||||
timestamp = response.headers['Last-Modified']
|
||||
else:
|
||||
timestamp = response.headers['Date']
|
||||
lastmod = time.strptime(timestamp, "%a, %d %b %Y %H:%M:%S %Z")
|
||||
item['lastmod'] = time.strftime("%Y-%m-%dT%H:%M:%S%z", lastmod)
|
||||
return item
|
||||
|
|
Loading…
Reference in New Issue