static: provide git services

This creates the redirect sites

 git.airshipit.org
 git.openstack.org
 git.starlingx.io
 git.zuul-ci.org

The htaccess rules are put into the main configuration file to avoid
having to create a directory and manage another file.  We use a macro
to duplicate the rules and retain the old semantics of the http site
redirecting directly (as opposed to doing a extra 301 to
https://git.openstack.org first).  This required adding "/" to the "^"
matches as it now runs in VirtualHost context; no functional change is
intended over the old sites.

This will require _acme-challenge CNAMEs to acme.opendev.org before
being merged.

testinfra is updated to exercise some redirects matching against the
results of the extant sites.

Change-Id: Iaa9d5dc2af3f5f8abc11c2312e4308b50f5fcd2b

playbooks/host_vars/static01.opendev.org.yaml

@@ -13,6 +13,14 @@ letsencrypt_certs:
     - docs.openstack.org
   static01-docs-starlingx-io:
     - docs.starlingx.io
+  static01-git-airshipit-org:
+    - git.airshipit.org
+  static01-git-openstack-org:
+    - git.openstack.org
+  static01-git-starlingx-io:
+    - git.starlingx.io
+  static01-git-zuul-ci-org:
+    - git.zuul-ci.org
   static01-governance-openstack-org:
     - governance.openstack.org
   static01-service-types-openstack-org:

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -50,6 +50,18 @@
 - name: letsencrypt updated static01-docs-starlingx-io
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 
+- name: letsencrypt updated static01-git-airshipit-org
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
+- name: letsencrypt updated static01-git-starlingx-io
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
+- name: letsencrypt updated static01-git-openstack-org
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
+- name: letsencrypt updated static01-git-zuul-ci-org
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
 - name: letsencrypt updated static01-governance-openstack-org
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 

playbooks/roles/static/tasks/enable_git_site.yaml

@@ -0,0 +1,14 @@
+- name: 'Add git site {{ hostname }}'
+  template:
+    src: '50-git.conf.j2'
+    dest: '/etc/apache2/sites-available/50-{{ hostname }}.conf'
+    owner: root
+    group: root
+    mode: 0644
+
+- name: 'Enable {{ hostname }}'
+  command: 'a2ensite 50-{{ hostname }}'
+  args:
+    creates: '/etc/apache2/sites-enabled/50-{{ hostname }}'
+  notify:
+    - Reload apache2

playbooks/roles/static/tasks/main.yaml

@@ -77,3 +77,13 @@
     - 50-tarballs.opendev.org
     - 50-tarballs.openstack.org
     - 50-zuul-ci.org
+
+- name: Enable git sites
+  include_tasks: enable_git_site.yaml
+  loop:
+    - git.airshipit.org
+    - git.openstack.org
+    - git.starlingx.io
+    - git.zuul-ci.org
+  loop_control:
+    loop_var: hostname

playbooks/roles/static/templates/50-git.conf.j2

@@ -0,0 +1,42 @@
+#
+# This file is generated and managed by Ansible
+#
+
+{% include 'git-redirects.conf.j2' %}
+
+##
+# vhost configuration
+##
+
+<VirtualHost *:80>
+  ServerName {{ hostname }}
+
+  Use GitRedirects
+
+  LogLevel warn
+  ErrorLog /var/log/apache2/{{ hostname }}_error.log
+  CustomLog /var/log/apache2/{{ hostname }}_access.log combined
+  ServerSignature Off
+</VirtualHost>
+
+<IfModule mod_ssl.c>
+<VirtualHost *:443>
+  ServerName {{ hostname }}
+
+  SSLCertificateFile      /etc/letsencrypt-certs/{{ hostname }}/{{ hostname }}.cer
+  SSLCertificateKeyFile   /etc/letsencrypt-certs/{{ hostname }}/{{ hostname }}.key
+  SSLCertificateChainFile /etc/letsencrypt-certs/{{ hostname }}/ca.cer
+  SSLProtocol All -SSLv2 -SSLv3
+  # Note: this list should ensure ciphers that provide forward secrecy
+  SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+  SSLHonorCipherOrder on
+
+  Use GitRedirects
+
+  ErrorLog /var/log/apache2/{{ hostname }}_error.log
+  CustomLog /var/log/apache2/{{ hostname }}_access.log combined
+  ServerSignature Off
+</VirtualHost>
+</IfModule>
+
+UndefMacro GitRedirects

playbooks/roles/static/templates/git-redirects.conf.j2

@@ -0,0 +1,192 @@
+<Macro GitRedirects>
+
+RewriteEngine On
+
+# Unescape any slashes in the branch portion of the query string so
+# that we don't have to worry about whether or not they are encoded
+# later.  This is a recursive rule to handle multiple slashes.
+RewriteCond %{QUERY_STRING} "^(.*)h=([^&]+)%2F([^&]+)(.*)$"
+RewriteRule "^/(.*)$" "/$1?%1h=%2/%3%4" [N]
+
+# Map whitelabeled project git sites with repository prefixes
+RewriteCond %{HTTP_HOST} ^git\.airshipit\.org$ [NC]
+RewriteRule "^/(cgit/|)(airship-.*)$" "/$1openstack/$2"
+RewriteCond %{HTTP_HOST} ^git\.starlingx\.io$ [NC]
+RewriteRule "^/(cgit/|)(stx-.*)$" "/$1openstack/$2"
+RewriteCond %{HTTP_HOST} ^git\.zuul-ci\.org$ [NC]
+RewriteRule "^/(cgit/|)((nodepool|zuul).*)$" "/$1openstack-infra/$2"
+
+###################################
+# summary
+#  ignore all args
+
+RewriteRule "^/cgit/([^/]+)/([^/]+)/?$" "https://opendev.org/$1/$2/" [L,QSD]
+
+###################################
+# refs tab -> branches tab
+#  ignore all args
+# The cgit refs tab shows tags+branches, the branches tab in gitea is
+# the closest linkable url
+
+RewriteRule "^/cgit/([^/]+)/([^/]+)/refs/?" "https://opendev.org/$1/$2/branches" [L,QSD]
+
+###################################
+# log tab (with file) -> commits tab
+#  h=
+
+RewriteCond %{QUERY_STRING} h=([\w/]+)
+RewriteRule "^/cgit/([^/]+)/([^/]+)/log/?(.*)" "https://opendev.org/$1/$2/commits/branch/%1/$3" [L,QSD]
+
+#  no args
+
+RewriteRule "^/cgit/([^/]+)/([^/]+)/log/?(.*)" "https://opendev.org/$1/$2/commits/branch/master/$3" [L,QSD]
+
+#####################################################
+# tree tab (with file) -> tree tab
+#  id=
+#  h=&id= (id)
+# if there's a commit, it takes precedence
+RewriteCond %{QUERY_STRING} id=([\w]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/tree/?(.*)" "https://opendev.org/$1/$2/src/commit/%1/$3" [L,QSD]
+
+#  h=
+# we have a commit pointed for a head
+RewriteCond %{QUERY_STRING} h=([0-9a-f]{40})
+RewriteRule "^/cgit/(.*?)/(.*?)/tree/?(.*)" "https://opendev.org/$1/$2/src/commit/%1/$3" [L,QSD]
+
+#  h=
+# if there's no commit, but a branch:
+RewriteCond %{QUERY_STRING} h=([\w/]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/tree/?(.*)" "https://opendev.org/$1/$2/src/branch/%1/$3" [L,QSD]
+
+# if there's no args:
+RewriteRule "^/cgit/(.*?)/(.*?)/tree/?(.*)" "https://opendev.org/$1/$2/src/branch/master/$3" [L,QSD]
+
+#####################################################
+# plain link without file -> tree tab
+#  id=
+#  h=&id= (id)
+# if there's a commit, it takes precedence
+RewriteCond %{QUERY_STRING} id=([\w]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/plain/?$" "https://opendev.org/$1/$2/src/commit/%1/$3" [L,QSD]
+
+#  h=
+# we have a commit pointed for a head
+RewriteCond %{QUERY_STRING} h=([0-9a-f]{40})
+RewriteRule "^/cgit/(.*?)/(.*?)/plain/?$" "https://opendev.org/$1/$2/src/commit/%1/$3" [L,QSD]
+
+#  h=
+# if there's no commit, but a branch:
+RewriteCond %{QUERY_STRING} h=([\w/]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/plain/?$" "https://opendev.org/$1/$2/src/branch/%1/$3" [L,QSD]
+
+# if there's no args:
+RewriteRule "^/cgit/(.*?)/(.*?)/plain/?$" "https://opendev.org/$1/$2/src/branch/master/$3" [L,QSD]
+
+#####################################################
+# plain link (with file) -> raw
+#  same as tree
+#  id=
+#  h=&id= (id)
+# if there's a commit, it takes precedence
+RewriteCond %{QUERY_STRING} id=([\w]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/plain/?(.*)" "https://opendev.org/$1/$2/raw/commit/%1/$3" [L,QSD]
+
+#  h=
+# we have a commit pointed for a head
+RewriteCond %{QUERY_STRING} h=([0-9a-f]{40})
+RewriteRule "^/cgit/(.*?)/(.*?)/plain/?(.*)$" "https://opendev.org/$1/$2/raw/commit/%1/$3" [L,QSD]
+
+#  h=
+# if there's no commit, but a branch:
+RewriteCond %{QUERY_STRING} h=([\w/]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/plain/?(.*)" "https://opendev.org/$1/$2/raw/branch/%1/$3" [L,QSD]
+
+# if there's no args:
+RewriteRule "^cgit/(.*?)/(.*?)/plain/?(.*)" "https://opendev.org/$1/$2/raw/branch/master/$3" [L,QSD]
+
+######################
+# commit tab (with file) -> commit screen (without file)
+#  id=
+#  id=&h=
+RewriteCond %{QUERY_STRING} id=([\w]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/commit/?(.*)" "https://opendev.org/$1/$2/commit/%1" [L,QSD]
+
+#  h=
+# we have a commit pointed for a head
+RewriteCond %{QUERY_STRING} h=([0-9a-f]{40})
+RewriteRule "^/cgit/(.*?)/(.*?)/commit/?(.*)" "https://opendev.org/$1/$2/commit/%1" [L,QSD]
+
+#  h=
+# The commit tab in cgit will show the branch-tip commit in this case.
+# There is not a comprable page in gitea, so we redirect to the branch
+# log (which has the branch-tip commit at the top of the list).  We
+# include the file if it's there to further restrict the list of
+# commits
+RewriteCond %{QUERY_STRING} h=([\w/]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/commit/?(.*)" "https://opendev.org/$1/$2/commits/branch/%1/$3" [L,QSD]
+
+#  no args
+# Same, but with master branch
+RewriteRule "^/cgit/(.*?)/(.*?)/commit/?(.*)" "https://opendev.org/$1/$2/commits/branch/master/$3" [L,QSD]
+
+######################
+# diff (with file) -> commit screen (without file)
+# Gitea doesn't handle arbitrary diffs, so just show the commit page for id.
+# We do the same thing as for the commit tab.
+#  id=&id2=
+#  id=
+#  id=&h=
+RewriteCond %{QUERY_STRING} id=([\w]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/diff/?(.*)" "https://opendev.org/$1/$2/commit/%1" [L,QSD]
+
+#  h=
+# we have a commit pointed for a head
+RewriteCond %{QUERY_STRING} h=([0-9a-f]{40})
+RewriteRule "^/cgit/(.*?)/(.*?)/diff/?(.*)" "https://opendev.org/$1/$2/commit/%1" [L,QSD]
+
+#  h=
+RewriteCond %{QUERY_STRING} h=([\w/]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/diff/?(.*)" "https://opendev.org/$1/$2/commits/branch/%1/$3" [L,QSD]
+
+#  no args
+RewriteRule "^/cgit/(.*?)/(.*?)/diff/?(.*)" "https://opendev.org/$1/$2/commits/branch/master/$3" [L,QSD]
+
+######################
+# patch (with file)
+# Gitea doesn't handle generating patch files, so just show the commit page.
+# We do the same thing as for the commit tab.
+#  id=
+#  id=&h=
+RewriteCond %{QUERY_STRING} id=([\w]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/patch/?(.*)" "https://opendev.org/$1/$2/commit/%1" [L,QSD]
+
+#  h=
+# we have a commit pointed for a head
+RewriteCond %{QUERY_STRING} h=([0-9a-f]{40})
+RewriteRule "^/cgit/(.*?)/(.*?)/patch/?(.*)" "https://opendev.org/$1/$2/commit/%1" [L,QSD]
+
+#  h=
+RewriteCond %{QUERY_STRING} h=([\w/]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/patch/?(.*)" "https://opendev.org/$1/$2/commits/branch/%1/$3" [L,QSD]
+
+#  no args
+RewriteRule "^/cgit/(.*?)/(.*?)/patch/?(.*)" "https://opendev.org/$1/$2/commits/branch/master/$3" [L,QSD]
+
+#####################
+# tag
+# Gitea doesn't have a dedicated tag page, but if you click a tag in
+# gitea, it takes you to the source tree view for that tag, which has
+# the tagged commit at the top of the table.
+RewriteCond %{QUERY_STRING} h=([\w/\.]+)
+RewriteRule "^/cgit/(.*?)/(.*?)/tag/?" "https://opendev.org/$1/$2/src/tag/%1" [L,QSD]
+
+#####################
+# Any other unknown cgit url, redirect to /
+RewriteRule "^/cgit" "https://opendev.org/" [L,QSD]
+
+#####################
+# Non cgit URLs
+RewriteRule "^/(.*)$" "https://opendev.org/$1" [L]
+
+</Macro>

testinfra/test_static.py

@@ -129,3 +129,28 @@ def test_zuulci_org(host, name):
                    '--resolve %s:443:127.0.0.1 https://%s/ ' %
                    (name, name))
     assert 'Zuul is an open source CI tool' in cmd.stdout
+
+# test redirects as they are.  leave http off the first one and we
+# test both http/https.
+git_redirects = (
+    ('git.openstack.org/openstack/nova', 'https://opendev.org/openstack/nova'),
+    ('git.openstack.org/cgit/openstack/tripleo-ansible/commit/?id=a6f9b1551baf5f680c05f4fa69ac926f8a0a3f81',
+     'https://opendev.org/openstack/tripleo-ansible/commit/a6f9b1551baf5f680c05f4fa69ac926f8a0a3f81'),
+    ('git.starlingx.io/stx-tools', 'https://opendev.org/openstack/stx-tools'),
+    ('git.zuul-ci.org/zuul', 'https://opendev.org/openstack-infra/zuul'),
+    ('git.airshipit.org/airship-in-a-bottle', 'https://opendev.org/openstack/airship-in-a-bottle')
+)
+@pytest.mark.parametrize("url,target", git_redirects)
+
+def test_git_redirects(host, url, target):
+    hostname = url.split('/')[0]
+
+    # http should redirect directly (not bounce via https)
+    cmd = host.run('curl --resolve %s:80:127.0.0.1 http://%s' % (hostname, url))
+    assert '302 Found' in cmd.stdout
+    assert target in cmd.stdout
+
+    cmd = host.run('curl --insecure --resolve %s:443:127.0.0.1 https://%s' %
+                   (hostname, url))
+    assert '302 Found' in cmd.stdout
+    assert target in cmd.stdout