[sitemap] do not add empty URLs to the list of start URLs

This will solve the 'ValueError: Missing scheme in request url'
exception.

Change-Id: I295b2a519668b36413ec2ba4ef3ba9dfaac6e859
This commit is contained in:
Christian Berendt 2016-01-22 14:58:44 +01:00
parent b4f4013423
commit 44243238ac
1 changed files with 3 additions and 1 deletions

View File

@ -58,8 +58,10 @@ class SitemapSpider(spiders.CrawlSpider):
super(SitemapSpider, self).__init__(*args, **kwargs)
self.domain = domain
self.allowed_domains = [domain]
self.start_urls = ['http://%s/index.html' % domain]
self.start_urls = ['http://%s' % domain]
for url in urls.split(','):
if not url:
continue
self.start_urls.append(url)
def parse_item(self, response):