Ansible playbook to generate I18n Extra-ATCs

Change-Id: Idbfadc365f9bf0e3dcf3b03b51b0862dafefa7e9

ansible-requirement.txt

@@ -0,0 +1,18 @@
+# WARNING:
+# This file is use by all Ansible roles for testing purposes.
+
+# PURPOSE:
+# Python requirements listed here are imported by the roles via tox
+# target configuration in each role.
+
+# The Ansible version used for testing
+ansible==2.3.0.0
+
+# The Ansible lint version used for lint tests
+ansible-lint==3.4.18
+
+# Used for the ip filter within ansible
+netaddr
+
+# Used for json_query
+jmespath==0.9.3

playbooks/README.rst

@@ -0,0 +1,21 @@
+Ansible playbooks for I18n maintenance
+======================================
+
+This folder contains Ansible playbooks for maintenance I18n work.
+Typically the playbooks are running localy on user's workstation.
+Ansible >= 2.2.0.0 is required to work with json_query.
+
+Usage: "ansible-playbook <playbook.yml>"
+
+More convenient way is to use **tox** like
+   ``tox -e ansible -- ansible-playbook <playbook.yml>``.
+
+generate_atc.yml
+----------------
+
+This playbook uses the output of zanata_stats.py, zanata_userinfo.py
+and zanata_users.py and generates a list of Extra-ATCS in the target
+formats for the proposal to the governance repo and the Wiki page:
+
+http://git.openstack.org/cgit/openstack/governance/tree/reference/projects.yaml
+https://wiki.openstack.org/wiki/I18nTeam/ATC_statistics

playbooks/atc.json.j2

@@ -0,0 +1,4 @@
+    - name: {{ username }}
+      email: {{ useremail }}
+      expires-in: {{ userexpires }}
+      comment: {{ usercomment }}

playbooks/generate_atc.yml

@@ -0,0 +1,100 @@
+---
+# Ansible playbook to generate extra-atc snippets for projects.yaml in
+# governance repo.
+# Requires 3 source files
+# - userstatsfile: generated output from zanata_stats.py
+#                  (default: zanata_stats_output.csv)
+# - userinfofile: generated output from zanata_userinfo.py
+#                  (default: zanata_userinfo_output.csv)
+# - the current projects.yaml file from governance repo
+#   https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
+#
+# minimum translation count is to configure (default 300 words/30
+# phrases)
+#
+# Old extra-atc list will be reviewed and refreshed with new data
+#
+# usage: ansible-playbook generate_atc.yml
+#
+# The output can be found in /tmp/atc.stats
+- hosts: localhost
+  gather_facts: "no"
+  vars:
+    userstatsfile: "zanata_stats_output.csv"
+    userinfofile: "zanata_userinfo_output.csv"
+    translatecount: 300
+    workingdir: "/tmp/atc.output"
+  vars_files:
+    - "projects.yaml"
+
+  tasks:
+    - name: create working dir
+      file:
+        path: "{{ workingdir }}"
+        state: directory
+
+    - name: read old atc list
+      vars:
+        extraatcs: "{{ I18n['extra-atcs'] }}"
+        useremail: "{{ item.email }}"
+        username: "{{ item.name }}"
+        usercomment: "{{ item.comment }}"
+        userexpires: "{{ item['expires-in'] }}"
+        expiredate: "28 {{ userexpires[:3] }}{{ userexpires[-5:] }}"
+        udateuser: "{{lookup('pipe','LANG=en date -d \"' + expiredate + '\" \"+%s\"')}}"
+        udatecur: "{{lookup('pipe','LANG=en date -d \"+1 month\" \"+%s\"') }}"
+      template:
+        src: atc.json.j2
+        dest: "{{ workingdir }}/user.{{ usercomment.split(' ')[0] }}"
+      when: udateuser|int > udatecur|int
+      with_items:
+        - "{{ extraatcs }}"
+
+    - name: read user stats
+      vars:
+        userdata: "{{ item.split(',') }}"
+        useremail: "{{ lookup('csvfile', userdata[0] + ' file=' + userinfofile + ' delimiter=, col=4') }}"
+        username: "{{ lookup('csvfile', userdata[0] + ' file=' + userinfofile + ' delimiter=, col=3') }}"
+        userlang: "{{ lookup('csvfile', userdata[0] + ' file=' + userinfofile + ' delimiter=, col=1') }}"
+        usercomment: "{{ userdata[0] }} ({{ userlang }})"
+        userexpires: "{{lookup('pipe','LANG=en date \"+%B %Y\" -d \"+1 year\"')}}"
+      template:
+        src: atc.json.j2
+        dest: "{{ workingdir }}/user.{{ userdata[0] }}"
+        force: "yes"
+      with_lines: cat "{{ userstatsfile }}"
+      when: userdata[4]|int + userdata[9]|int >= translatecount
+
+    - name: validate openstackid
+      vars:
+        userdata: "{{ item.split(',') }}"
+        useremail: "{{ lookup('csvfile', userdata[0] + ' file=' + userinfofile + ' delimiter=, col=4') }}"
+        zanataid: "{{ userdata[0] }}"
+      uri:
+        url: "https://openstackid-resources.openstack.org/api/public/v1/members?expand=groups&relations=all_affiliations,groups&filter[]=email=={{ useremail }}"
+        method: GET
+        return_content: yes
+        validate_certs: no
+        headers:
+          Content-Type: "application/json"
+          Accept: "application/json"
+      register: api
+      with_lines: cat "{{ userstatsfile }}"
+      when: userdata[4]|int + userdata[9]|int >= translatecount
+
+    - name: write wiki stats files per user
+      template:
+        src: stats.json.j2
+        dest: "{{ workingdir }}/stats.{{ item|hash('sha1') }} "
+      with_items: "{{ api|json_query('results[].{ item: item, userdata: json.data[].{email_verified: email_verified, groups: groups[?id==`5`].code|[0]}}') }}"
+
+    - name: create stats file proposal
+      shell: cat "{{ workingdir }}"/user.* > /tmp/atc.stats
+
+    - name: create stats file wiki
+      shell: cat "{{ workingdir }}"/stats.* > /tmp/atc.wiki
+
+    - name: delete working dir
+      file:
+        path: "{{ workingdir }}"
+        state: absent

playbooks/stats.json.j2

@@ -0,0 +1,10 @@
+|-
+|{{ item|
+replace(",","||")|
+replace("'","")|
+replace("{item: u","")|
+replace(" userdata: [{","")|
+replace(" groups: ufoundation-members}]}","foundation-member")|
+replace(" userdata: []}","no foundation-member")|
+replace("userdata: None}","no foundation-member")|
+replace("groups: None}]}","no foundation-member") }}||

playbooks/tests/ansible.cfg

@@ -0,0 +1,2 @@
+[defaults]
+roles_path = ../..

tox.ini

@@ -10,6 +10,15 @@ setenv = VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 
+[testenv:ansible]
+deps =
+    {[testenv]deps}
+    -r{toxinidir}/ansible-requirement.txt
+passenv = HOME
+setenv =
+  ANSIBLE_CONFIG = {toxinidir}/playbook/tests/ansible.cfg
+commands = {posargs}
+
 [testenv:venv]
 commands = {posargs}