Browse Source

[sitemap] do not add empty URLs to the list of start URLs

This will solve the 'ValueError: Missing scheme in request url'
exception.

Change-Id: I295b2a519668b36413ec2ba4ef3ba9dfaac6e859
tags/0.33.0
Christian Berendt 3 years ago
parent
commit
44243238ac
1 changed files with 3 additions and 1 deletions
  1. +3
    -1
      sitemap/generator/spiders/sitemap.py

+ 3
- 1
sitemap/generator/spiders/sitemap.py View File

@@ -58,8 +58,10 @@ class SitemapSpider(spiders.CrawlSpider):
super(SitemapSpider, self).__init__(*args, **kwargs)
self.domain = domain
self.allowed_domains = [domain]
self.start_urls = ['http://%s/index.html' % domain]
self.start_urls = ['http://%s' % domain]
for url in urls.split(','):
if not url:
continue
self.start_urls.append(url)

def parse_item(self, response):

Loading…
Cancel
Save