Ansible job definitions for OpenStack projects CI systems
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

propose_translation_update.sh 10KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. #!/bin/bash -xe
  2. # Licensed under the Apache License, Version 2.0 (the "License"); you may
  3. # not use this file except in compliance with the License. You may obtain
  4. # a copy of the License at
  5. #
  6. # http://www.apache.org/licenses/LICENSE-2.0
  7. #
  8. # Unless required by applicable law or agreed to in writing, software
  9. # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  10. # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  11. # License for the specific language governing permissions and limitations
  12. # under the License.
  13. PROJECT=$1
  14. BRANCH=$2
  15. JOBNAME=$3
  16. HORIZON_DIR=$4
  17. # Replace /'s in the branch name with -'s because Zanata does not
  18. # allow /'s in version names.
  19. ZANATA_VERSION=${BRANCH//\//-}
  20. SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
  21. source $SCRIPTSDIR/common_translation_update.sh
  22. init_branch $BRANCH
  23. function cleanup_module {
  24. local modulename=$1
  25. # Remove obsolete files.
  26. cleanup_po_files "$modulename"
  27. cleanup_pot_files "$modulename"
  28. # Compress downloaded po files, this needs to be done after
  29. # cleanup_po_files since that function needs to have information the
  30. # number of untranslated strings.
  31. compress_po_files "$modulename"
  32. }
  33. # Add all po files to the git repo in a target directory
  34. function git_add_po_files {
  35. local target_dir=$1
  36. local po_file_count
  37. po_file_count=`find $1 -name *.po | wc -l`
  38. if [ $po_file_count -ne 0 ]; then
  39. git add $target_dir/*/*
  40. fi
  41. }
  42. # Add all JSON files to the git repo (used in javascript translations)
  43. function git_add_json_files {
  44. local target_dir=$1
  45. local json_file_count
  46. json_file_count=`find $1 -name '*.json' | wc -l`
  47. if [ $json_file_count -ne 0 ]; then
  48. git add $target_dir/*
  49. fi
  50. }
  51. # Propose updates for manuals
  52. function propose_manuals {
  53. # Pull updated translations from Zanata.
  54. pull_from_zanata "$PROJECT"
  55. # Compress downloaded po files
  56. case "$PROJECT" in
  57. openstack-manuals)
  58. # Cleanup po and pot files
  59. cleanup_module "doc"
  60. ;;
  61. api-site)
  62. # Cleanup po and pot files
  63. cleanup_module "api-quick-start"
  64. cleanup_module "firstapp"
  65. ;;
  66. security-doc)
  67. cleanup_module "security-guide"
  68. ;;
  69. esac
  70. # Add imported upstream translations to git
  71. for FILE in ${DocFolder}/*; do
  72. DOCNAME=${FILE#${DocFolder}/}
  73. if [ -d ${DocFolder}/${DOCNAME}/locale ] ; then
  74. git_add_po_files ${DocFolder}/${DOCNAME}/locale
  75. fi
  76. if [ -d ${DocFolder}/${DOCNAME}/source/locale ] ; then
  77. git_add_po_files ${DocFolder}/${DOCNAME}/source/locale
  78. fi
  79. done
  80. }
  81. # Propose updates for training-guides
  82. function propose_training_guides {
  83. # Pull updated translations from Zanata.
  84. pull_from_zanata "$PROJECT"
  85. # Cleanup po and pot files
  86. cleanup_module "doc/upstream-training"
  87. # Add all changed files to git
  88. git_add_po_files doc/upstream-training/source/locale
  89. }
  90. # Propose updates for i18n
  91. function propose_i18n {
  92. # Pull updated translations from Zanata.
  93. pull_from_zanata "$PROJECT"
  94. # Cleanup po and pot files
  95. cleanup_module "doc"
  96. # Add all changed files to git
  97. git_add_po_files doc/source/locale
  98. }
  99. # Propose updates for python and django projects
  100. function propose_python_django {
  101. local modulename=$1
  102. local version=$2
  103. # Check for empty directory and exit early
  104. local content
  105. content=$(ls -A $modulename/locale/)
  106. if [[ "$content" == "" ]] ; then
  107. return
  108. fi
  109. # Now add all changed files to git.
  110. # Note we add them here to not have to differentiate in the functions
  111. # between new files and files already under git control.
  112. git_add_po_files $modulename/locale
  113. # Cleanup po and pot files
  114. cleanup_module "$modulename"
  115. if [ "$version" == "master" ] ; then
  116. # Remove not anymore translated log files on master, but not
  117. # on released stable branches.
  118. cleanup_log_files "$modulename"
  119. fi
  120. # Check first whether directory exists, it might be missing if
  121. # there are no translations.
  122. if [[ -d "$modulename/locale/" ]] ; then
  123. # Some files were changed, add changed files again to git, so
  124. # that we can run git diff properly.
  125. git_add_po_files $modulename/locale
  126. fi
  127. }
  128. # Handle either python or django proposals
  129. function handle_python_django_project {
  130. local project=$1
  131. setup_project "$project" "$ZANATA_VERSION"
  132. pull_from_zanata "$project"
  133. handle_python_django $project python
  134. handle_python_django $project django
  135. handle_project_doc $project
  136. }
  137. # Handle project doc proposals
  138. function handle_project_doc {
  139. local project=$1
  140. # Doing only things in the test repos for project doc translation
  141. if ! [[ ${DOC_TARGETS[*]} =~ "$project" ]]; then
  142. return
  143. fi
  144. # setup_project and pull_from_zanata are already done
  145. # we start directly with generating .pot files
  146. extract_messages_doc
  147. # cleanup po and pot files
  148. cleanup_module "doc"
  149. # Add all changed files to git
  150. git_add_po_files doc/source/locale
  151. }
  152. # Handle either python or django proposals
  153. function handle_python_django {
  154. local project=$1
  155. # kind can be "python" or "django"
  156. local kind=$2
  157. local module_names
  158. module_names=$(get_modulename $project $kind)
  159. if [ -n "$module_names" ]; then
  160. if [[ "$kind" == "django" ]] ; then
  161. install_horizon
  162. fi
  163. propose_releasenotes "$ZANATA_VERSION"
  164. for modulename in $module_names; do
  165. # Note that we need to generate the pot files so that we
  166. # can calculate how many strings are translated.
  167. case "$kind" in
  168. django)
  169. # Update the .pot file
  170. extract_messages_django "$modulename"
  171. ;;
  172. python)
  173. # Extract messages from project except log messages
  174. extract_messages_python "$modulename"
  175. ;;
  176. esac
  177. propose_python_django "$modulename" "$ZANATA_VERSION"
  178. done
  179. fi
  180. }
  181. function propose_releasenotes {
  182. local version=$1
  183. # This function does not check whether releasenote publishing and
  184. # testing are set up in zuul/layout.yaml. If releasenotes exist,
  185. # they get pushed to the translation server.
  186. # Note that releasenotes only get translated on master.
  187. if [[ "$version" == "master" && -f releasenotes/source/conf.py ]]; then
  188. # Note that we need to generate these so that we can calculate
  189. # how many strings are translated.
  190. extract_messages_releasenotes "keep_workdir"
  191. local lang_po
  192. local locale_dir=releasenotes/source/locale
  193. for lang_po in $(find $locale_dir -name 'releasenotes.po'); do
  194. check_releasenotes_per_language $lang_po
  195. done
  196. # Remove the working directory. We no longer needs it.
  197. rm -rf releasenotes/work
  198. # Cleanup POT files.
  199. # PO files are already clean up in check_releasenotes_translations.
  200. cleanup_pot_files "releasenotes"
  201. # Compress downloaded po files, this needs to be done after
  202. # cleanup_po_files since that function needs to have information the
  203. # number of untranslated strings.
  204. compress_po_files "releasenotes"
  205. # Add all changed files to git - if there are
  206. # translated files at all.
  207. if [ -d releasenotes/source/locale/ ] ; then
  208. git_add_po_files releasenotes/source/locale
  209. fi
  210. fi
  211. # Remove any releasenotes translations from stable branches, they
  212. # are not needed there.
  213. if [[ "$version" != "master" && -d releasenotes/source/locale ]]; then
  214. # Note that content might exist, e.g. from downloaded translations,
  215. # but are not under git control.
  216. git rm --ignore-unmatch -rf releasenotes/source/locale
  217. fi
  218. }
  219. function propose_reactjs {
  220. pull_from_zanata "$PROJECT"
  221. # Clean up files (removes incomplete translations and untranslated strings)
  222. cleanup_module "i18n"
  223. # Convert po files to ReactJS i18n JSON format
  224. for lang in `find i18n/*.po -printf "%f\n" | sed 's/\.po$//'`; do
  225. npm run po2json -- ./i18n/$lang.po -o ./i18n/$lang.json
  226. # The files are created as a one-line JSON file - expand them
  227. python -m json.tool ./i18n/$lang.json ./i18n/locales/$lang.json
  228. rm ./i18n/$lang.json
  229. done
  230. # Add JSON files to git
  231. git_add_json_files i18n/locales
  232. }
  233. # Setup git repository for git review.
  234. setup_git
  235. # Check whether a review already exists, setup review commit message.
  236. # Function setup_review calls setup_commit_message which will set CHANGE_ID and
  237. # CHANGE_NUM if a change exists and will always set COMMIT_MSG.
  238. setup_review "$BRANCH"
  239. # If a change already exists, let's pull it in and compute the
  240. # 'git patch-id' of it.
  241. PREV_PATCH_ID=""
  242. if [[ -n ${CHANGE_NUM} ]]; then
  243. # Ignore errors we get in case we can't download the patch with
  244. # git-review. If that happens then we will submit a new patch.
  245. set +e
  246. git review -d ${CHANGE_NUM}
  247. RET=$?
  248. if [[ "$RET" -eq 0 ]]; then
  249. PREV_PATCH_ID=$(git show | git patch-id | awk '{print $1}')
  250. fi
  251. set -e
  252. # The git review changed our branch, go back to our correct branch
  253. git checkout -f ${BRANCH}
  254. fi
  255. # Setup venv - needed for all projects for subunit
  256. setup_venv
  257. case "$PROJECT" in
  258. api-site|openstack-manuals|security-doc)
  259. init_manuals "$PROJECT"
  260. setup_manuals "$PROJECT" "$ZANATA_VERSION"
  261. propose_manuals
  262. propose_releasenotes "$ZANATA_VERSION"
  263. ;;
  264. training-guides)
  265. setup_training_guides "$ZANATA_VERSION"
  266. propose_training_guides
  267. ;;
  268. i18n)
  269. setup_i18n "$ZANATA_VERSION"
  270. propose_i18n
  271. ;;
  272. tripleo-ui)
  273. setup_reactjs_project "$PROJECT" "$ZANATA_VERSION"
  274. propose_reactjs
  275. ;;
  276. *)
  277. # Common setup for python and django repositories
  278. handle_python_django_project $PROJECT
  279. ;;
  280. esac
  281. # Filter out commits we do not want.
  282. filter_commits
  283. # Propose patch to gerrit if there are changes.
  284. send_patch "$BRANCH"
  285. if [ $INVALID_PO_FILE -eq 1 ] ; then
  286. echo "At least one po file in invalid. Fix all invalid files on the"
  287. echo "translation server."
  288. exit 1
  289. fi
  290. # Tell finish function that everything is fine.
  291. ERROR_ABORT=0