From 7500ab080524a6332ad4a1ea3c9a24c87f33785e Mon Sep 17 00:00:00 2001 From: Frank Kloeker Date: Sun, 7 Jan 2018 14:52:46 +0100 Subject: [PATCH] Ansible playbook to generate I18n Extra-ATCs Change-Id: Idbfadc365f9bf0e3dcf3b03b51b0862dafefa7e9 --- ansible-requirement.txt | 18 +++++++ playbooks/README.rst | 21 ++++++++ playbooks/atc.json.j2 | 4 ++ playbooks/generate_atc.yml | 100 ++++++++++++++++++++++++++++++++++++ playbooks/stats.json.j2 | 10 ++++ playbooks/tests/ansible.cfg | 2 + tox.ini | 9 ++++ 7 files changed, 164 insertions(+) create mode 100644 ansible-requirement.txt create mode 100644 playbooks/README.rst create mode 100644 playbooks/atc.json.j2 create mode 100644 playbooks/generate_atc.yml create mode 100644 playbooks/stats.json.j2 create mode 100644 playbooks/tests/ansible.cfg diff --git a/ansible-requirement.txt b/ansible-requirement.txt new file mode 100644 index 0000000..12d2def --- /dev/null +++ b/ansible-requirement.txt @@ -0,0 +1,18 @@ +# WARNING: +# This file is use by all Ansible roles for testing purposes. + +# PURPOSE: +# Python requirements listed here are imported by the roles via tox +# target configuration in each role. + +# The Ansible version used for testing +ansible==2.3.0.0 + +# The Ansible lint version used for lint tests +ansible-lint==3.4.18 + +# Used for the ip filter within ansible +netaddr + +# Used for json_query +jmespath==0.9.3 diff --git a/playbooks/README.rst b/playbooks/README.rst new file mode 100644 index 0000000..7fd92e6 --- /dev/null +++ b/playbooks/README.rst @@ -0,0 +1,21 @@ +Ansible playbooks for I18n maintenance +====================================== + +This folder contains Ansible playbooks for maintenance I18n work. +Typically the playbooks are running localy on user's workstation. +Ansible >= 2.2.0.0 is required to work with json_query. + +Usage: "ansible-playbook " + +More convenient way is to use **tox** like + ``tox -e ansible -- ansible-playbook ``. + +generate_atc.yml +---------------- + +This playbook uses the output of zanata_stats.py, zanata_userinfo.py +and zanata_users.py and generates a list of Extra-ATCS in the target +formats for the proposal to the governance repo and the Wiki page: + +http://git.openstack.org/cgit/openstack/governance/tree/reference/projects.yaml +https://wiki.openstack.org/wiki/I18nTeam/ATC_statistics diff --git a/playbooks/atc.json.j2 b/playbooks/atc.json.j2 new file mode 100644 index 0000000..56549f1 --- /dev/null +++ b/playbooks/atc.json.j2 @@ -0,0 +1,4 @@ + - name: {{ username }} + email: {{ useremail }} + expires-in: {{ userexpires }} + comment: {{ usercomment }} diff --git a/playbooks/generate_atc.yml b/playbooks/generate_atc.yml new file mode 100644 index 0000000..73b33bd --- /dev/null +++ b/playbooks/generate_atc.yml @@ -0,0 +1,100 @@ +--- +# Ansible playbook to generate extra-atc snippets for projects.yaml in +# governance repo. +# Requires 3 source files +# - userstatsfile: generated output from zanata_stats.py +# (default: zanata_stats_output.csv) +# - userinfofile: generated output from zanata_userinfo.py +# (default: zanata_userinfo_output.csv) +# - the current projects.yaml file from governance repo +# https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml +# +# minimum translation count is to configure (default 300 words/30 +# phrases) +# +# Old extra-atc list will be reviewed and refreshed with new data +# +# usage: ansible-playbook generate_atc.yml +# +# The output can be found in /tmp/atc.stats +- hosts: localhost + gather_facts: "no" + vars: + userstatsfile: "zanata_stats_output.csv" + userinfofile: "zanata_userinfo_output.csv" + translatecount: 300 + workingdir: "/tmp/atc.output" + vars_files: + - "projects.yaml" + + tasks: + - name: create working dir + file: + path: "{{ workingdir }}" + state: directory + + - name: read old atc list + vars: + extraatcs: "{{ I18n['extra-atcs'] }}" + useremail: "{{ item.email }}" + username: "{{ item.name }}" + usercomment: "{{ item.comment }}" + userexpires: "{{ item['expires-in'] }}" + expiredate: "28 {{ userexpires[:3] }}{{ userexpires[-5:] }}" + udateuser: "{{lookup('pipe','LANG=en date -d \"' + expiredate + '\" \"+%s\"')}}" + udatecur: "{{lookup('pipe','LANG=en date -d \"+1 month\" \"+%s\"') }}" + template: + src: atc.json.j2 + dest: "{{ workingdir }}/user.{{ usercomment.split(' ')[0] }}" + when: udateuser|int > udatecur|int + with_items: + - "{{ extraatcs }}" + + - name: read user stats + vars: + userdata: "{{ item.split(',') }}" + useremail: "{{ lookup('csvfile', userdata[0] + ' file=' + userinfofile + ' delimiter=, col=4') }}" + username: "{{ lookup('csvfile', userdata[0] + ' file=' + userinfofile + ' delimiter=, col=3') }}" + userlang: "{{ lookup('csvfile', userdata[0] + ' file=' + userinfofile + ' delimiter=, col=1') }}" + usercomment: "{{ userdata[0] }} ({{ userlang }})" + userexpires: "{{lookup('pipe','LANG=en date \"+%B %Y\" -d \"+1 year\"')}}" + template: + src: atc.json.j2 + dest: "{{ workingdir }}/user.{{ userdata[0] }}" + force: "yes" + with_lines: cat "{{ userstatsfile }}" + when: userdata[4]|int + userdata[9]|int >= translatecount + + - name: validate openstackid + vars: + userdata: "{{ item.split(',') }}" + useremail: "{{ lookup('csvfile', userdata[0] + ' file=' + userinfofile + ' delimiter=, col=4') }}" + zanataid: "{{ userdata[0] }}" + uri: + url: "https://openstackid-resources.openstack.org/api/public/v1/members?expand=groups&relations=all_affiliations,groups&filter[]=email=={{ useremail }}" + method: GET + return_content: yes + validate_certs: no + headers: + Content-Type: "application/json" + Accept: "application/json" + register: api + with_lines: cat "{{ userstatsfile }}" + when: userdata[4]|int + userdata[9]|int >= translatecount + + - name: write wiki stats files per user + template: + src: stats.json.j2 + dest: "{{ workingdir }}/stats.{{ item|hash('sha1') }} " + with_items: "{{ api|json_query('results[].{ item: item, userdata: json.data[].{email_verified: email_verified, groups: groups[?id==`5`].code|[0]}}') }}" + + - name: create stats file proposal + shell: cat "{{ workingdir }}"/user.* > /tmp/atc.stats + + - name: create stats file wiki + shell: cat "{{ workingdir }}"/stats.* > /tmp/atc.wiki + + - name: delete working dir + file: + path: "{{ workingdir }}" + state: absent diff --git a/playbooks/stats.json.j2 b/playbooks/stats.json.j2 new file mode 100644 index 0000000..b112d13 --- /dev/null +++ b/playbooks/stats.json.j2 @@ -0,0 +1,10 @@ +|- +|{{ item| +replace(",","||")| +replace("'","")| +replace("{item: u","")| +replace(" userdata: [{","")| +replace(" groups: ufoundation-members}]}","foundation-member")| +replace(" userdata: []}","no foundation-member")| +replace("userdata: None}","no foundation-member")| +replace("groups: None}]}","no foundation-member") }}|| diff --git a/playbooks/tests/ansible.cfg b/playbooks/tests/ansible.cfg new file mode 100644 index 0000000..6c8a344 --- /dev/null +++ b/playbooks/tests/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +roles_path = ../.. diff --git a/tox.ini b/tox.ini index 82f921f..c9742c6 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,15 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt +[testenv:ansible] +deps = + {[testenv]deps} + -r{toxinidir}/ansible-requirement.txt +passenv = HOME +setenv = + ANSIBLE_CONFIG = {toxinidir}/playbook/tests/ansible.cfg +commands = {posargs} + [testenv:venv] commands = {posargs}