Browse Source

[sitemap] do not add empty URLs to the list of start URLs

This will solve the 'ValueError: Missing scheme in request url'
exception.

Change-Id: I295b2a519668b36413ec2ba4ef3ba9dfaac6e859
tags/0.33.0
Christian Berendt 3 years ago
parent
commit
44243238ac
1 changed files with 3 additions and 1 deletions
  1. 3
    1
      sitemap/generator/spiders/sitemap.py

+ 3
- 1
sitemap/generator/spiders/sitemap.py View File

@@ -58,8 +58,10 @@ class SitemapSpider(spiders.CrawlSpider):
58 58
         super(SitemapSpider, self).__init__(*args, **kwargs)
59 59
         self.domain = domain
60 60
         self.allowed_domains = [domain]
61
-        self.start_urls = ['http://%s/index.html' % domain]
61
+        self.start_urls = ['http://%s' % domain]
62 62
         for url in urls.split(','):
63
+            if not url:
64
+                continue
63 65
             self.start_urls.append(url)
64 66
 
65 67
     def parse_item(self, response):

Loading…
Cancel
Save