opendev
/
system-config
Code Issues Proposed changes

Merge "Translate gitea project creation to python"

This commit is contained in:
Zuul 2019-07-16 19:31:11 +00:00 committed by Gerrit Code Review
parent 4e050d981e caebf387b4
commit 36344bfcdd
8 changed files with 190 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
playbooks/group_vars/gitea.yaml
View File

@ -1,3 +1,4 @@
ansible_python_interpreter: python3
gitea_root_email: infra-root@openstack.org
gitea_gerrit_public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVuhTMAz1H2Jr9AC3py9A0vlNna6Sdt4yrvZOayxukPqQ7GPZd+Mo7MVyypxLD479N2mA09JAdsbq1eTiPP8ksEkB+dNxZzw8mY1653R/IXSW6J9xPcoDa88HF2s/xHN24IWzgiDjNNe79AQ+sKleByEQZ++xXny3MRpy258hKUvAtjjOLOnM1PBs8JNOzBL+UPgWRgSX6GG0qywJZqjD1Qx5kvH9RTRLi+tcMhEi4laN7BYvn4csY0sYzTzPG4ZTu3ootIJoRlQGtQ0LmoFO1vSwyEJUags6/ZZGjgy3jl3kwcU/b8ZnFlF4MDw1OB1QqMb4r6bMHbXNIupp4zJbz gerrit-replication-2014-04-25
iptables_extra_public_tcp_ports:

0
playbooks/roles/gitea-git-repos/library/__init__.py Normal file
View File

177
playbooks/roles/gitea-git-repos/library/gitea_create_repos.py Executable file
View File

@ -0,0 +1,177 @@
#!/usr/bin/env python3
#
# Copyright 2019 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
import requests
import urllib.parse
from ansible.module_utils.basic import AnsibleModule
SB_REPO = 'https://storyboard.openstack.org/#!/project/{org}/{repo}'
SB_FORMAT = 'https://storyboard.openstack.org/#!/story/{{index}}'
LP_REPO = 'https://bugs.launchpad.net/{repo}'
LP_FORMAT = 'https://bugs.launchpad.net/{repo}/+bug/{{index}}'
class Gitea(object):
def __init__(self, url, password, always_update, projects):
self.url = url
self.password = password
self.always_update = always_update
self.projects = projects
self.orgs = { f['project'].split('/')[0] for f in self.projects }
def request(self, method, endpoint, *args, **kwargs):
resp = requests.request(
method,
urllib.parse.urljoin(self.url, endpoint),
auth=('root', self.password),
verify=False,
*args, **kwargs)
resp.raise_for_status()
return resp
def get(self, endpoint, *args, **kwargs):
return self.request('GET', endpoint, *args, **kwargs)
def post(self, endpoint, *args, **kwargs):
return self.request('POST', endpoint, *args, **kwargs)
def put(self, endpoint, *args, **kwargs):
return self.request('PUT', endpoint, *args, **kwargs)
def get_gitea_orgs(self):
orgs = self.get("/api/v1/user/orgs").json()
return [f['username'] for f in orgs]
def make_gitea_org(self, org):
self.post(
'/api/v1/admin/users/root/orgs',
json=dict(username=org))
def ensure_gitea_teams(self, org):
team_list = self.get('/api/v1/orgs/{org}/teams'.format(org=org)).json()
owner_id = [f['id'] for f in team_list if f['name'] == 'Owners'][0]
org_owners = self.get(
'/api/v1/teams/{owner_id}/members'.format(owner_id=owner_id))
if 'gerrit' not in [f['username'] for f in org_owners.json()]:
self.put('/api/v1/teams/{owner_id}/members/gerrit'.format(
owner_id=owner_id))
def get_org_repo_list(self, org):
return self.get('/api/v1/orgs/{org}/repos'.format(org=org)).json()
def get_csrf_token(self):
resp = self.get('/')
return urllib.parse.unquote(resp.cookies.get('_csrf'))
def make_gitea_project(self, project, csrf_token):
org, repo = project['project'].split('/', 1)
resp = self.post(
'/api/v1/org/{org}/repos'.format(org=org),
json=dict(
auto_init=True,
description=project.get('description', '')[:255],
name=repo,
private=False,
readme='Default'))
if project.get('use-storyboard'):
external_tracker_url = SB_REPO.format(org=org, repo=repo)
tracker_url_format = SB_FORMAT
elif project.get('groups'):
external_tracker_url = LP_REPO.format(repo=project['groups'][0])
tracker_url_format = LP_FORMAT.format(repo=project['groups'][0])
else:
external_tracker_url = LP_REPO.format(repo=repo)
tracker_url_format = LP_FORMAT.format(repo=repo)
self.post(
'/{org}/{repo}/settings'.format(org=org, repo=repo),
data=dict(
_csrf=csrf_token,
action='advanced',
# enable_pulls is not provided, which disables it
# enable_wiki is not provided, which disables it
enable_external_wiki=False,
external_wiki_url='',
# enable_issues is on so that issue links work
enable_issues='on',
enable_external_tracker=True,
external_tracker_url=external_tracker_url,
tracker_url_format=tracker_url_format,
tracker_issue_style='numeric',
))
for count in range(0, 5):
try:
return self.post(
'/{org}/{repo}/settings/branches'.format(
org=org, repo=repo),
data=dict(
_csrf=csrf_token,
action='default_branch',
branch='master',
))
except requests.exceptions.HTTPError as e:
time.sleep(3)
raise Exception("Could not update branch settings")
def run(self):
gitea_orgs = self.get_gitea_orgs()
gitea_repos = []
for org in self.orgs:
if org not in gitea_orgs:
self.make_gitea_org(org)
self.ensure_gitea_teams(org)
gitea_repos.extend(self.get_org_repo_list(org))
csrf_token = self.get_csrf_token()
for project in self.projects:
if project['project'] not in gitea_repos or self.always_update:
self.make_gitea_project(project, csrf_token)
def ansible_main():
module = AnsibleModule(
argument_spec=dict(
url=dict(required=True),
password=dict(required=True),
projects=dict(required=True, type='list'),
always_update=dict(type='bool', default=True),
)
)
p = module.params
gitea = Gitea(
url=p.get('url'),
password=p.get('password'),
always_update=p.get('always_update'),
projects=p.get('projects'),
)
try:
gitea.run()
except Exception as e:
module.fail_json(msg=str(e), changed=True)
module.exit_json(changed=True)
if __name__ == '__main__':
ansible_main()

50
playbooks/roles/gitea-git-repos/tasks/main.yaml
View File

@ -1,44 +1,8 @@
- name: Get Gerrit project list
set_fact:
gerrit_projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}"
- name: Parse Gerrit org list
set_fact:
gerrit_orgs: "{{ gerrit_projects | map(attribute='project') | map('regex_search', '^(.*?)/') | list | unique | select | map('regex_replace', '/', '') | list }}"
- name: debug
debug:
msg: "{{ gerrit_orgs }}"
- name: Get Gitea org list
# We assume that all the orgs we are interested in are owned by root
uri:
url: "{{ gitea_url }}/api/v1/user/orgs"
user: root
- name: Create Gitea Repos and Org
gitea_create_repos:
url: "{{ gitea_url }}"
password: "{{ gitea_root_password }}"
force_basic_auth: true
validate_certs: false
status_code: 200
register: gitea_org_list
- name: Parse Gitea org list
set_fact:
gitea_orgs: "{{ gitea_org_list.json | map(attribute='username') | list }}"
- name: Create orgs
loop: "{{ gerrit_orgs }}"
loop_control:
loop_var: org
include_tasks: 'setup-org.yaml'
- name: Get a CSRF token
uri:
url: "{{ gitea_url }}/"
validate_certs: false
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
register: gitea_token
- name: Parse CSRF taken
set_fact:
gitea_token: "{{ gitea_token.cookies._csrf|regex_replace('%3D','=') }}"
- name: Create repos
loop: "{{ gerrit_projects }}"
loop_control:
loop_var: project
include_tasks: 'setup-repo.yaml'
when: gitea_always_update or project.project not in gitea_repos
always_update: "{{ gitea_always_update }}"
# Lookup runs locally on the calling machine, so doesn't need
# /opt/project-config remotely
projects: "{{ lookup('file', '/opt/project-config/gerrit/projects.yaml') | from_yaml }}"

56
playbooks/roles/gitea-git-repos/tasks/setup-org.yaml
View File

@ -1,56 +0,0 @@
- name: Process org
debug:
msg: "Processing org {{ org }}"
- name: Create org
when: org not in gitea_orgs
uri:
url: "{{ gitea_url }}/api/v1/admin/users/root/orgs"
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
validate_certs: false
status_code: 201
method: POST
body_format: json
body:
username: "{{ org }}"
- name: Get org team list
uri:
url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/teams"
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
validate_certs: false
status_code: 200
register: gitea_org_team_list
- name: Get org owners
uri:
url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members"
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
validate_certs: false
status_code: 200
register: gitea_org_members
- name: Add Gerrit user to org
when: "'gerrit' not in gitea_org_members.json | map(attribute='username')"
uri:
url: "{{ gitea_url }}/api/v1/teams/{{ (gitea_org_team_list.json | selectattr('name', 'equalto', 'Owners') | list)[0]['id'] }}/members/gerrit"
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
validate_certs: false
status_code: 204
method: PUT
- name: Get org repo list
uri:
url: "{{ gitea_url }}/api/v1/orgs/{{ org }}/repos"
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
validate_certs: false
status_code: 200
register: gitea_org_repo_list
- name: Parse org repo list
set_fact:
gitea_repos: "{{ gitea_org_repo_list.json | map(attribute='full_name') | list + gitea_repos | default([]) }}"

98
playbooks/roles/gitea-git-repos/tasks/setup-repo.yaml
View File

@ -1,98 +0,0 @@
- name: debug
debug:
msg: "{{ project }}"
- name: Parse project name
set_fact:
org: "{{ project.project | regex_replace('^(.*)/(.*)$', '\\1') }}"
repo: "{{ project.project | regex_replace('^(.*)/(.*)$', '\\2') }}"
- name: Create repo
when: project.project not in gitea_repos
uri:
url: "{{ gitea_url }}/api/v1/org/{{ org }}/repos"
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
validate_certs: false
status_code: 201
method: POST
body_format: json
body:
auto_init: true
description: "{{ (project.description | default(''))[:255] }}"
name: "{{ repo }}"
private: false
readme: Default
register: create_repo_result
- name: Set storyboard tracker url
when: "'use-storyboard' in project and project['use-storyboard']"
set_fact:
external_tracker_url: "https://storyboard.openstack.org/#!/project/{{ org }}/{{ repo }}"
- name: Set storyboard tracker url format
when: "'use-storyboard' in project and project['use-storyboard']"
set_fact:
tracker_url_format: "https://storyboard.openstack.org/#!/story/{index}"
- name: Set launchpad tracker url
when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' not in project or not project['groups'])"
set_fact:
external_tracker_url: "https://bugs.launchpad.net/{{ repo }}"
- name: Set launchpad tracker url format
when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' not in project or not project['groups'])"
set_fact:
tracker_url_format: "https://bugs.launchpad.net/{{ repo }}/+bug/{index}"
- name: Set launchpad tracker url if group set
when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' in project and project['groups'])"
set_fact:
external_tracker_url: "https://bugs.launchpad.net/{{ project.groups[0] }}"
- name: Set launchpad tracker url format if group set
when: "('use-storyboard' not in project or not project['use-storyboard']) and ('groups' in project and project['groups'])"
set_fact:
tracker_url_format: "https://bugs.launchpad.net/{{ project.groups[0] }}/+bug/{index}"
- name: Adjust repo settings
when: gitea_always_update or project.project not in gitea_repos
register: result
retries: 3
until: result is succeeded
delay: 5
uri:
url: "{{ gitea_url }}/{{ org }}/{{ repo }}/settings"
validate_certs: false
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
status_code: 302
method: POST
body_format: form-urlencoded
body:
_csrf: "{{ gitea_token }}"
action: advanced
# enable_pulls is not provided, which disables it
# enable_wiki is not provided, which disables it
enable_external_wiki: false
external_wiki_url:
# enable_issues is on so that issue links work
enable_issues: on
enable_external_tracker: true
external_tracker_url: "{{ external_tracker_url }}"
tracker_url_format: "{{ tracker_url_format }}"
tracker_issue_style: numeric
- name: Set default branch
when: gitea_always_update or project.project not in gitea_repos
register: result
retries: 3
until: result is succeeded
delay: 5
uri:
url: "{{ gitea_url }}/{{ org }}/{{ repo }}/settings/branches"
validate_certs: false
user: root
password: "{{ gitea_root_password }}"
force_basic_auth: true
status_code: 302
method: POST
body_format: form-urlencoded
body:
_csrf: "{{ gitea_token }}"
action: default_branch
branch: master

5
playbooks/roles/gitea/tasks/main.yaml
View File

@ -32,6 +32,11 @@
template:
src: app.ini.j2
dest: /var/gitea/conf/app.ini
- name: Install requests
package:
name:
- python3-requests
state: present
- name: Install docker-compose
package:
name:

2
playbooks/sync-gitea-projects.yaml
View File

@ -8,7 +8,6 @@
repo: https://git.openstack.org/openstack-infra/project-config
dest: /opt/project-config
force: yes
register: gitinfo
- hosts: "gitea:!disabled"
name: "Create repos on gitea servers"
@ -16,5 +15,4 @@
max_fail_percentage: 1
roles:
- role: gitea-git-repos
project_config_ref: "{{ hostvars.localhost.gitinfo.after }}"
gitea_always_update: true