Tools used by OpenStack Documentation
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

123456789101112131415161718192021222324252627282930313233
  1. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  2. # not use this file except in compliance with the License. You may obtain
  3. # a copy of the License at
  4. #
  5. # http://www.apache.org/licenses/LICENSE-2.0
  6. #
  7. # Unless required by applicable law or agreed to in writing, software
  8. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  9. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  10. # License for the specific language governing permissions and limitations
  11. # under the License.
  12. # Configuration variables used inside Scrapy to enable modules/pipelines
  13. # and to affect the behavior of several parts.
  14. from scrapy import linkextractors
  15. BOT_NAME = 'sitemap'
  16. SPIDER_MODULES = ['generator.spiders']
  17. ITEM_PIPELINES = {
  18. 'generator.pipelines.IgnoreDuplicateUrls': 500,
  19. 'generator.pipelines.ExportSitemap': 100,
  20. }
  21. CONCURRENT_REQUESTS = 32
  22. CONCURRENT_REQUESTS_PER_DOMAIN = 32
  23. CONCURRENT_REQUESTS_PER_IP = 32
  24. DOWNLOAD_WARNSIZE = 67108864
  25. LOG_LEVEL = 'INFO'
  26. LOGGING_ENABLED = True
  27. RANDOMIZE_DOWNLOAD_DELAY = False
  28. ROBOTSTXT_OBEY = True
  29. TELNETCONSOLE_ENABLED = False
  30. linkextractors.IGNORED_EXTENSIONS.remove('pdf')