Improve sitemap generation

Filter out any trunk links Change-Id: Icf35370237e0209c836ed8883040bf6fe4c9a7c0
2014-06-27 11:04:38 +02:00 · 2014-06-27 11:04:38 +02:00 · a6fbc259b6
commit a6fbc259b6
parent 0ec4225119
1 changed files with 3 additions and 0 deletions
--- a/sitemap/generator/spiders/sitemap.py
+++ b/sitemap/generator/spiders/sitemap.py
@ -31,6 +31,9 @@ class SitemapSpider(spiders.CrawlSpider):
                    r'.*\.xml',
                    r'.*\.txt',
                    r'.*/',
+                ],
+                deny=[
+                    r'/trunk/',
                ]
            ),
            follow=True, callback='parse_item'