gitea-image: add a robots.txt

This looks like a very sane default robots.txt.  We can modify it as
required.

Change-Id: I8b9d3aa63538388e319f0216535f7a1d977f4885

docker/gitea/custom/public/robots.txt

@@ -0,0 +1,35 @@
+# This was kindly seeded with a mix of
+#  https://git.lelux.fi/theel0ja/gitea-robots.txt/src/branch/master/robots.txt
+# and
+#  https://github.com/robots.txt
+# at 2020-07-01
+#
+# Some commented out items are left to indicate we have considered
+# them and would like to explicitly allow them for indexing while they
+# are not causing problems.
+
+User-agent: *
+
+# Disallow: /avatars
+# Disallow: /user/*
+# Disallow: /*/*/src/commit/*
+# Disallow: /*/*/commit/*
+
+Disallow: /*/*/activity/*
+Disallow: /vendor/librejs.html
+Disallow: /api/swagger
+Disallow: /swagger.*.json
+
+# Language spam
+Disallow: /*?lang=
+
+# From github
+Disallow: */archive/
+Disallow: */blame/
+# Disallow: /raw/*
+Disallow: /.git/
+Disallow: */.git/
+Disallow: /*.git$
+Disallow: /*q=
+
+Crawl-delay: 2

testinfra/test_gitea.py

@@ -33,3 +33,9 @@ def test_ulimit(host):
 def test_sshd_logs(host):
     cmd = host.run("docker logs gitea-docker_gitea-ssh_1")
     assert cmd.stdout != '' or cmd.stderr != ''
+
+def test_robots(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve gitea99.opendev.org:3000:127.0.0.1 '
+                   'https://gitea99.opendev.org:3000/robots.txt')
+    assert 'Disallow: /' in cmd.stdout