Add check for old-style nic config files

Since old-style nic config files can no longer be used in Queens
or Rocky (the format changed in Ocata), add a check in yaml-validate
to detect that old-style files are in use and list conversion script
that can be used.

Made some changes to the script to ask before overwriting the nic
config file and save a datestamped copy as backup.  In addition,
the script now takes an optional parameter to define location
of run-os-net-config.sh.

Change-Id: Ic56c48fa35ab2f4c1762c0e370be03fbf2e7671c
Closes-Bug: 1753812

releasenotes/notes/check-old-style-nic-config-4624a60e3303411b.yaml

@@ -0,0 +1,5 @@
+---
+other:
+  - Add check for nic config files using the old style format
+    (os-apply-config) and list the script that can be used
+    to convert the file.

tools/yaml-nic-config-2-script.py

@@ -11,15 +11,31 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import argparse
 import collections
 import copy
+import datetime
 import os
+import shutil
 import sys
 import traceback
 import yaml
 import six
 import re
 
+def parse_opts(argv):
+    parser = argparse.ArgumentParser(
+        description='Convert an old style NIC config file into the new format using '
+                    'run-os-net-config.sh')
+    parser.add_argument('--script-dir', metavar='<script directory>',
+                        help="Relative path to run-os-net-config.sh",
+                        default="network/scripts/run-os-net-config.sh")
+    parser.add_argument('files', nargs="+", metavar='<file>',
+                        help='List of one or more NIC config files to convert')
+    opts = parser.parse_args(argv[1:])
+
+    return opts
+
 
 #convert comments into 'comments<num>: ...' YAML
 def to_commented_yaml(filename):
@@ -65,6 +81,8 @@ def to_normal_yaml(filename):
     out_str = ''
     next_line_break = False
     for line in data.split('\n'):
+        # get_input not supported by run-os-net-config.sh script
+        line = line.replace('get_input: ', '')
         m = re.match(" +comment[0-9]+_([0-9]+): '(.*)'.*", line) #normal comments
         i = re.match(" +inline_comment[0-9]+: '(.*)'.*", line) #inline comments
         if m:
@@ -131,11 +149,7 @@ def write_template(template, filename=None):
     with open(filename, 'w') as f:
         yaml.dump(template, f, TemplateDumper, width=120, default_flow_style=False)
 
-def exit_usage():
+def convert(filename, script_path):
-    print('Usage %s <yaml file>' % sys.argv[0])
-    sys.exit(1)
-
-def convert(filename):
     print('Converting %s' % filename)
     try:
         tpl = yaml.load(open(filename).read(), Loader=TemplateLoader)
@@ -143,20 +157,6 @@ def convert(filename):
         print(traceback.format_exc())
         return 0
 
-    # Check which path we need for run-os-net-config.sh because we have
-    # nic config templates in the top-level and network/config
-    script_paths = ['network/scripts/run-os-net-config.sh',
-                    '../../scripts/run-os-net-config.sh']
-    script_path = None
-    for p in script_paths:
-        check_path = os.path.join(os.path.dirname(filename), p)
-        if os.path.isfile(check_path):
-            print("Found %s, using %s" % (check_path, p))
-            script_path = p
-    if script_path is None:
-        print("Error couldn't find run-os-net-config.sh relative to filename")
-        exit_usage()
-
     for r in (tpl.get('resources', {})).items():
         if (r[1].get('type') == 'OS::Heat::StructuredConfig' and
             r[1].get('properties', {}).get('group') == 'os-apply-config' and
@@ -180,7 +180,7 @@ def convert(filename):
     # Preserve typical HOT template key ordering
     od_result = collections.OrderedDict()
     # Need to bump the HOT version so str_replace supports serializing to json
-    od_result['heat_template_version'] = "2016-10-14"
+    od_result['heat_template_version'] = "rocky"
     if tpl.get('description'):
         od_result['description'] = description(tpl['description'])
     od_result['parameters'] = tpl['parameters']
@@ -189,31 +189,76 @@ def convert(filename):
     #print('Result:')
     #print('%s' % yaml.dump(od_result, Dumper=TemplateDumper, width=120, default_flow_style=False))
     #print('---')
-    #replace = raw_input(
+
-        #"Replace file %s?  Answer y/n" % filename).lower() == 'y'
-    #if replace:
-    #print("Replace %s" % filename)
     write_template(od_result, filename)
-    #else:
+
-    #    print("NOT replacing %s" % filename)
-    #    return 0
     return 1
 
-if len(sys.argv) < 2:
+def check_old_style(filename):
-    exit_usage()
 
-path_args = sys.argv[1:]
+    with open(filename, 'r') as f:
+        tpl = yaml.load(open(filename).read())
+
+        if isinstance(tpl.get('resources', {}), dict):
+            for r in (tpl.get('resources', {})).items():
+                if (r[1].get('type') == 'OS::Heat::StructuredConfig' and
+                    r[1].get('properties', {}).get('group') == 'os-apply-config' and
+                    r[1].get('properties', {}).get('config', {}).get('os_net_config')):
+                    return True
+
+    return False
+
+opts = parse_opts(sys.argv)
 exit_val = 0
 num_converted = 0
 
-for base_path in path_args:
+for base_path in opts.files:
     if os.path.isfile(base_path) and base_path.endswith('.yaml'):
-        to_commented_yaml(base_path)
+
-        num_converted += convert(base_path)
+        if check_old_style(base_path):
-        to_normal_yaml(base_path)
+
+            # Check for script in the user entered (or default) location or in
+            # path relative to NIC config files
+            script_paths = [opts.script_dir]
+            script_paths.append('../../scripts/run-os-net-config.sh')
+            script_paths.append(
+'/usr/share/openstack-tripleo-heat-templates/network/scripts/run-os-net-config.sh')
+
+            script_path = None
+            for p in script_paths:
+                if os.path.isfile(os.path.join(os.path.dirname(base_path), p)):
+                    script_path = p
+                    break
+            if script_path is None:
+                print("Error couldn't find run-os-net-config.sh relative to filename")
+                sys.exit(1)
+
+            print("Using script at %s" % script_path)
+
+            extension = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
+            backup_filename = os.path.realpath(base_path) + '.' + extension
+
+            print('The yaml file will be overwritten and the original saved as %s'
+                  % backup_filename)
+            if not raw_input("Overwrite %s? [y/n] " % base_path).lower() == 'y':
+                print("Skipping file %s" % base_path)
+                continue
+
+            if os.path.exists(backup_filename):
+                print("Backup file already exists, skipping file %s" % base_path)
+                continue
+
+            shutil.copyfile(base_path, backup_filename)
+
+            to_commented_yaml(base_path)
+            num_converted += convert(base_path, script_path)
+            to_normal_yaml(base_path)
+        else:
+            print('File %s is not using old style NIC configuration' % base_path)
+
     else:
         print('Unexpected argument %s' % base_path)
-        exit_usage()
+
 if num_converted == 0:
   exit_val = 1
 sys.exit(exit_val)

tools/yaml-validate.py

@@ -721,6 +721,10 @@ def validate(filename, param_map):
         if filename.startswith('./network_data_'):
             retval = validate_network_data_file(filename)
 
+        if retval == 0 and is_heat_template:
+            # check for old style nic config files
+            retval = validate_nic_config_file(filename, tpl)
+
     except Exception:
         print(traceback.format_exc())
         return 1
@@ -792,6 +796,22 @@ def validate_network_data_file(data_file_path):
         return 1
     return 0
 
+def validate_nic_config_file(filename, tpl):
+    try:
+        if isinstance(tpl.get('resources', {}), dict):
+            for r in (tpl.get('resources', {})).items():
+                if (r[1].get('type') == 'OS::Heat::StructuredConfig' and
+                    r[1].get('properties', {}).get('group') == 'os-apply-config' and
+                    r[1].get('properties', {}).get('config', {}).get('os_net_config')):
+                    print('ERROR: Using old format of nic configuration file: %s' % filename)
+                    print('tools/yaml-nic-config-2-script.py can be used to convert to new format')
+                    return 1
+
+    except Exception:
+        print(traceback.format_exc())
+        return 1
+    return 0
+
 def parse_args():
     p = argparse.ArgumentParser()