Improve the autohelp scripts

wrapper:
- Don't modify the openstack-manuals repo if it exists, keep it as is
  git wise (no branch change, no pull)
- Install oslo.i18n and hplefthandclient
- First setup the environment, then do the update/docbook work

autohelp.py:
- Handle projects not using oslo.i18n (mostly for havana)

diff_branches.py:
- Update the output format
- Setup the venv for the requested projects only

Change-Id: I48c45a56f4b7b7298fef0b7595d62e445c92df39

RELEASE_NOTES.rst

@@ -6,7 +6,10 @@ Release notes
 
 * ``openstack-doc-test``: Optimize translation imports, improve output
   messages.
-* ``autohelp.py``: Improve sanitizer.
+* ``autohelp.py``: Improve sanitizer, better support for i18n in projects.
+* ``autohelp-wrapper``: Smarter handling of the manuals repo and environment
+  setup.
+* ``diff_branches.py``: Updated output format.
 
 0.18.1
 ------

autogenerate_config_docs/autohelp-wrapper

@@ -40,7 +40,7 @@ usage() {
 }
 
 setup_venv() {
-    if [ ! -e $VENVDIR ]; then
+    if [ ! -e $VENVDIR/bin/activate ]; then
         virtualenv $VENVDIR
     fi
     . $VENVDIR/bin/activate
@@ -52,7 +52,9 @@ get_project() {
     if [ ! -e $SOURCESDIR/$project ]; then
         git clone $GITBASE/$project $SOURCESDIR/$project
     else
-        (cd $SOURCESDIR/$project && git pull)
+        if [ $project != openstack-manuals ]; then
+            (cd $SOURCESDIR/$project && git pull)
+        fi
     fi
 }
 
@@ -63,6 +65,9 @@ setup_tools() {
     (cd $SOURCESDIR/oslo-incubator && python setup.py install)
     pip install "GitPython>=0.3.2.RC1"
 
+    # autohelp.py requires this
+    pip install oslo.i18n
+
     # For some reason the ceilometer installation fails without these 2
     # packages pre-installed
     pip install setuptools pbr
@@ -77,7 +82,7 @@ setup_tools() {
 
     # Cinder
     # hp3parclient is needed for icehouse, but not for juno
-    pip install hp3parclient
+    pip install hp3parclient hplefthandclient
 
     # Neutron
     pip install ryu
@@ -139,9 +144,6 @@ fi
 setup_venv
 setup_tools
 
-cd $MANUALSREPO
-git checkout $BRANCH
-
 for project in $PROJECTS; do
     get_project $project
 
@@ -150,9 +152,11 @@ for project in $PROJECTS; do
         git checkout $BRANCH
         python setup.py install
     )
+done
 
+for project in $PROJECTS; do
     if [ "$ACTION" = "setup" ]; then
-        continue
+        break
     fi
 
     if [ -d $MANUALSREPO/tools/autogenerate-config-flagmappings ]; then

autogenerate_config_docs/autohelp.py

@@ -88,10 +88,24 @@ def import_modules(repo_location, package_name, verbose=0):
     # the builtins contain the _ function.
     requirements = os.path.join(repo_location, 'requirements.txt')
     with open(requirements) as fd:
+        with_i18n = False
         for line in fd:
             if line.startswith('oslo.i18n'):
                 i18n.enable_lazy()
                 i18n.install(package_name)
+                with_i18n = True
+                break
+        if not with_i18n:
+            # NOTE(gpocentek): projects didn't use oslo.i18n on havana, and
+            # some imports fail because _ is not yet registered in the
+            # builtins. We try to import and setup the translation tools
+            # manually.
+            try:
+                modname = "%s.openstack.common.gettextutils" % package_name
+                module = importlib.import_module(modname)
+                module.install(package_name)
+            except Exception:
+                pass
 
     pkg_location = os.path.join(repo_location, package_name)
     for root, dirs, files in os.walk(pkg_location):

autogenerate_config_docs/diff_branches.py

@@ -38,6 +38,7 @@ def setup_venv(branch, novenvupdate):
     if not os.path.exists('venv'):
         os.mkdir('venv')
     args = ["./autohelp-wrapper", "-b", branch, "-e", dirname, "setup"]
+    args.extend(PROJECTS)
     if subprocess.call(args) != 0:
         print("Impossible to create the %s environment." % branch)
         sys.exit(1)
@@ -176,16 +177,16 @@ def format_option_name(name):
         section, name = name.split('/')
     except ValueError:
         # name without a section ('log_dir')
-        return "[DEFAULT]/%s" % name
+        return "[DEFAULT] %s" % name
 
     try:
         # If we're dealing with swift, we'll have a filename to extract
         # ('proxy-server|filter:tempurl/use')
         filename, section = section.split('|')
-        return "%s.conf: [%s]/%s" % (filename, section, name)
+        return "%s.conf: [%s] %s" % (filename, section, name)
     except ValueError:
         # section but no filename ('database/connection')
-        return "[%s]/%s" % (section, name)
+        return "[%s] %s" % (section, name)
 
 
 def generate_docbook(project, new_branch, old_list, new_list):