c499b57e16
We've noticed that our uwsgi queues are filling up and a lot of requests are being made to robots.txt which ends up 500/503 erroring. Add a robots.txt file which allows crawling of our lists and archives with a delay value in hopes this will cause bots to cache results and not fill up the queue with repetetive requests. Change-Id: I660d8d43f6b2d96663212d93ec48e67d86e9e761
69 lines
2.6 KiB
Python
69 lines
2.6 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from util import take_screenshots
|
|
|
|
testinfra_hosts = ['lists99.opendev.org']
|
|
|
|
def test_mariadb_listening(host):
|
|
mariadb_port = host.socket("tcp://127.0.0.1:3306")
|
|
assert mariadb_port.is_listening
|
|
|
|
def test_mailman3_web_listening(host):
|
|
mailman_web = host.socket("tcp://0.0.0.0:8000")
|
|
assert mailman_web.is_listening
|
|
mailman_uwsgi = host.socket("tcp://0.0.0.0:8080")
|
|
assert mailman_uwsgi.is_listening
|
|
|
|
def test_mailman3_core_listening(host):
|
|
mailman_rest = host.socket("tcp://127.0.0.1:8001")
|
|
assert mailman_rest.is_listening
|
|
mailman_lmtp = host.socket("tcp://127.0.0.1:8024")
|
|
assert mailman_lmtp.is_listening
|
|
|
|
def test_apache2_listening(host):
|
|
apache2_http = host.socket("tcp://0.0.0.0:80")
|
|
assert apache2_http.is_listening
|
|
apache2_https = host.socket("tcp://0.0.0.0:443")
|
|
assert apache2_https.is_listening
|
|
|
|
def test_robots(host):
|
|
cmd = host.run('curl --insecure '
|
|
'--resolve lists.opendev.org:443:127.0.0.1 '
|
|
'https://lists.opendev.org/robots.txt')
|
|
assert 'Disallow: /accounts/*' in cmd.stdout
|
|
assert 'Allow: /archives/*' in cmd.stdout
|
|
|
|
def test_mailman3_screenshots(host):
|
|
shots = (
|
|
("https://lists.opendev.org:443", None, "mm3-opendev-main.png"),
|
|
("https://lists.opendev.org:443/archives/",
|
|
None, "mm3-opendev-archives.png"),
|
|
("https://lists.opendev.org:443"
|
|
"/archives/list/service-discuss@lists.opendev.org/",
|
|
None, "mm3-opendev-list.png"),
|
|
("https://lists.opendev.org:443"
|
|
"/accounts/login/?next=/mailman3/lists/",
|
|
None, "mm3-opendev-login.png"),
|
|
("https://lists.openstack.org:443", None, "mm3-openstack-main.png"),
|
|
("https://lists.openstack.org:443/archives/",
|
|
None, "mm3-openstack-archives.png"),
|
|
("https://lists.openstack.org:443"
|
|
"/archives/list/openstack-discuss@lists.openstack.org/",
|
|
None, "mm3-openstack-list.png"),
|
|
("https://lists.openstack.org:443"
|
|
"/accounts/login/?next=/mailman3/lists/",
|
|
None, "mm3-openstack-login.png"),
|
|
)
|
|
|
|
take_screenshots(host, shots)
|