Browse Source

Add yaml-editor to utils for editing yaml-files on the go

Add setup changes to install yaml-editor with spyglass
Pawan Singh Pal 4 months ago
parent
commit
12e25206f3

+ 2
- 0
MANIFEST.in View File

@@ -0,0 +1,2 @@
1
+recursive-include spyglass/utils/editor/static *
2
+recursive-include spyglass/utils/editor/templates *

+ 1
- 0
setup.py View File

@@ -36,6 +36,7 @@ setup(
36 36
     entry_points={
37 37
         'console_scripts': [
38 38
             'spyglass=spyglass.spyglass:main',
39
+            'yaml-editor=spyglass.utils.editor.editor:main',
39 40
         ],
40 41
         'data_extractor_plugins':
41 42
         ['formation=spyglass.data_extractor.plugins.formation:FormationPlugin',

+ 1
- 0
spyglass/data_extractor/custom_exceptions.py View File

@@ -35,6 +35,7 @@ class NoSpecMatched(BaseError):
35 35
             self.specs))
36 36
         sys.exit(1)
37 37
 
38
+
38 39
 class MissingAttributeError(BaseError):
39 40
     pass
40 41
 

+ 1
- 2
spyglass/data_extractor/plugins/tugboat/excel_parser.py View File

@@ -19,8 +19,7 @@ import sys
19 19
 import yaml
20 20
 from openpyxl import load_workbook
21 21
 from openpyxl import Workbook
22
-from spyglass.data_extractor.custom_exceptions import
23
-    NoSpecMatched, )
22
+from spyglass.data_extractor.custom_exceptions import NoSpecMatched
24 23
 # from spyglass.data_extractor.custom_exceptions
25 24
 
26 25
 LOG = logging.getLogger(__name__)

+ 31
- 1
spyglass/parser/engine.py View File

@@ -15,9 +15,12 @@
15 15
 import copy
16 16
 import json
17 17
 import logging
18
+import os
18 19
 import pkg_resources
19 20
 import pprint
20 21
 import sys
22
+import tempfile
23
+
21 24
 import jsonschema
22 25
 import netaddr
23 26
 import yaml
@@ -81,6 +84,17 @@ class ProcessDataSource():
81 84
         LOG.debug("Genesis Node Details:\n{}".format(
82 85
             pprint.pformat(self.genesis_node)))
83 86
 
87
+    def _get_genesis_node_ip(self):
88
+        """ Returns the genesis node ip """
89
+        ip = '0.0.0.0'
90
+        LOG.info("Getting Genesis Node IP")
91
+        if not self.genesis_node:
92
+            self._get_genesis_node_details()
93
+        ips = self.genesis_node.get('ip', '')
94
+        if ips:
95
+            ip = ips.get('oam', '0.0.0.0')
96
+        return ip
97
+
84 98
     def _validate_intermediary_data(self, data):
85 99
         """ Validates the intermediary data before generating manifests.
86 100
 
@@ -347,11 +361,27 @@ class ProcessDataSource():
347 361
             f.write(yaml_file)
348 362
         f.close()
349 363
 
350
-    def generate_intermediary_yaml(self):
364
+    def generate_intermediary_yaml(self, edit_intermediary=False):
351 365
         """ Generating intermediary yaml """
352 366
         LOG.info("Start: Generate Intermediary")
353 367
         self._apply_design_rules()
354 368
         self._get_genesis_node_details()
369
+        # This will validate the extracted data from different sources.
355 370
         self._validate_intermediary_data(self.data)
371
+        if edit_intermediary:
372
+            self.edit_intermediary_yaml()
373
+            # This will check if user edited changes are in order.
374
+            self._validate_intermediary_data(self.data)
356 375
         self.intermediary_yaml = self.data
357 376
         return self.intermediary_yaml
377
+
378
+    def edit_intermediary_yaml(self):
379
+        """ Edit generated data using on browser """
380
+        LOG.info(
381
+            "edit_intermediary_yaml: Invoking web server for yaml editing")
382
+        with tempfile.NamedTemporaryFile(mode='r+') as file_obj:
383
+            yaml.safe_dump(self.data, file_obj, default_flow_style=False)
384
+            host = self._get_genesis_node_ip()
385
+            os.system('yaml-editor -f {0} -h {1}'.format(file_obj.name, host))
386
+            file_obj.seek(0)
387
+            self.data = yaml.safe_load(file_obj)

+ 8
- 1
spyglass/spyglass.py View File

@@ -57,6 +57,11 @@ LOG = logging.getLogger('spyglass')
57 57
     '-idir',
58 58
     type=click.Path(exists=True),
59 59
     help='The path where intermediary file needs to be generated')
60
+@click.option(
61
+    '--edit_intermediary/--no_edit_intermediary',
62
+    '-e/-nedit',
63
+    default=True,
64
+    help='Flag to let user edit intermediary')
60 65
 @click.option(
61 66
     '--generate_manifests',
62 67
     '-m',
@@ -96,6 +101,7 @@ def main(*args, **kwargs):
96 101
     # Extract user provided inputs
97 102
     generate_intermediary = kwargs['generate_intermediary']
98 103
     intermediary_dir = kwargs['intermediary_dir']
104
+    edit_intermediary = kwargs['edit_intermediary']
99 105
     generate_manifests = kwargs['generate_manifests']
100 106
     manifest_dir = kwargs['manifest_dir']
101 107
     intermediary = kwargs['intermediary']
@@ -176,7 +182,8 @@ def main(*args, **kwargs):
176 182
             data_extractor.site_data)
177 183
 
178 184
         LOG.info("Generate intermediary yaml")
179
-        intermediary_yaml = process_input_ob.generate_intermediary_yaml()
185
+        intermediary_yaml = process_input_ob.generate_intermediary_yaml(
186
+            edit_intermediary)
180 187
     else:
181 188
         LOG.info("Loading intermediary from user provided input")
182 189
         with open(intermediary, 'r') as intermediary_file:

+ 0
- 0
spyglass/utils/editor/__init__.py View File


+ 157
- 0
spyglass/utils/editor/editor.py View File

@@ -0,0 +1,157 @@
1
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the 'License');
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+# http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an 'AS IS' BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+
15
+
16
+import json
17
+import logging
18
+import os
19
+import sys
20
+
21
+import click
22
+import yaml
23
+
24
+from flask import Flask, request, render_template, send_from_directory
25
+from flask_bootstrap import Bootstrap
26
+
27
+
28
+app_path = os.path.dirname(os.path.abspath(__file__))
29
+app = Flask('Yaml Editor!',
30
+            template_folder=os.path.join(app_path, 'templates'),
31
+            static_folder=os.path.join(app_path, 'static'))
32
+Bootstrap(app)
33
+logging.getLogger('werkzeug').setLevel(logging.ERROR)
34
+LOG = app.logger
35
+
36
+
37
+@app.route('/favicon.ico')
38
+def favicon():
39
+    return send_from_directory(app.static_folder, 'favicon.ico')
40
+
41
+
42
+@app.route('/', methods=['GET', 'POST'])
43
+def index():
44
+    """Renders index page to edit provided yaml file."""
45
+    LOG.info('Rendering yaml file for editing')
46
+    with open(app.config['YAML_FILE']) as file_obj:
47
+        data = yaml.safe_load(file_obj)
48
+    return render_template('yaml.html',
49
+                           data=json.dumps(data),
50
+                           change_str=app.config['STRING_TO_CHANGE'])
51
+
52
+
53
+@app.route('/save', methods=['POST'])
54
+def save():
55
+    """Save current progress on file."""
56
+    LOG.info('Saving edited inputs from user to yaml file')
57
+    out = request.json.get('yaml_data')
58
+    with open(app.config['YAML_FILE'], 'w') as file_obj:
59
+        yaml.safe_dump(out, file_obj, default_flow_style=False)
60
+    return "Data saved successfully!"
61
+
62
+
63
+@app.route('/saveExit', methods=['POST'])
64
+def save_exit():
65
+    """Save current progress on file and shuts down the server."""
66
+    LOG.info('Saving edited inputs from user to yaml file and shutting'
67
+             ' down server')
68
+    out = request.json.get('yaml_data')
69
+    with open(app.config['YAML_FILE'], 'w') as file_obj:
70
+        yaml.safe_dump(out, file_obj, default_flow_style=False)
71
+    func = request.environ.get('werkzeug.server.shutdown')
72
+    if func:
73
+        func()
74
+    return "Saved successfully, Shutting down app! You may close the tab!"
75
+
76
+
77
+@app.errorhandler(404)
78
+def page_not_found(e):
79
+    """Serves 404 error."""
80
+    LOG.info('User tried to access unavailable page.')
81
+    return '<h1>404: Page not Found!</h1>'
82
+
83
+
84
+def run(*args, **kwargs):
85
+    """Starts the server."""
86
+    LOG.info('Initiating web server for yaml editing')
87
+    port = kwargs.get('port', None)
88
+    if not port:
89
+        port = 8161
90
+    app.run(host='0.0.0.0', port=port, debug=False)
91
+
92
+
93
+@click.command()
94
+@click.option(
95
+    '--file',
96
+    '-f',
97
+    required=True,
98
+    type=click.File(),
99
+    multiple=False,
100
+    help="Path with file name to the intermediary yaml file."
101
+)
102
+@click.option(
103
+    '--host',
104
+    '-h',
105
+    default='0.0.0.0',
106
+    type=click.STRING,
107
+    multiple=False,
108
+    help="Optional port parameter to run Flask on."
109
+)
110
+@click.option(
111
+    '--port',
112
+    '-p',
113
+    default=8161,
114
+    type=click.INT,
115
+    multiple=False,
116
+    help="Optional port parameter to run Flask on."
117
+)
118
+@click.option(
119
+    '--string',
120
+    '-s',
121
+    default='#CHANGE_ME',
122
+    type=click.STRING,
123
+    multiple=False,
124
+    help="Text which is required to be changed on yaml file."
125
+)
126
+def main(*args, **kwargs):
127
+    LOG.setLevel(logging.INFO)
128
+    LOG.info('Initiating yaml-editor')
129
+    try:
130
+        yaml.safe_load(kwargs['file'])
131
+    except yaml.YAMLError as e:
132
+        LOG.error('EXITTING - Please provide a valid yaml file.')
133
+        if hasattr(e, 'problem_mark'):
134
+            mark = e.problem_mark
135
+            LOG.error("Error position: ({0}:{1})".format(
136
+                mark.line + 1, mark.column + 1))
137
+        sys.exit(2)
138
+    except Exception:
139
+        LOG.error('EXITTING - Please provide a valid yaml file.')
140
+        sys.exit(2)
141
+    LOG.info("""
142
+
143
+##############################################################################
144
+
145
+Please go to http://{0}:{1}/ to edit your yaml file.
146
+
147
+##############################################################################
148
+
149
+    """.format(kwargs['host'], kwargs['port']))
150
+    app.config['YAML_FILE'] = kwargs['file'].name
151
+    app.config['STRING_TO_CHANGE'] = kwargs['string']
152
+    run(*args, **kwargs)
153
+
154
+
155
+if __name__ == '__main__':
156
+    """Invoked when used as a script."""
157
+    main()

+ 92
- 0
spyglass/utils/editor/static/app.js View File

@@ -0,0 +1,92 @@
1
+// Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+//     http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+
16
+// This file includes all the frond-end functionality being used for the
17
+// yaml editor application.
18
+
19
+
20
+/**
21
+ * Calls /save URL to save edit progress.
22
+ * @param  {String} data Stringified JSON data.
23
+ */
24
+function save(data) {
25
+    $.ajax({
26
+        type: 'POST',
27
+        url: '/save',
28
+        data: data,
29
+        success: function(res) {
30
+            setTimeout(function() { alert(res); }, 3);
31
+        },
32
+        contentType: 'application/json;charset=UTF-8'
33
+    });
34
+}
35
+
36
+/**
37
+ * Calls /saveExit URL to save edit progress and shut down web server.
38
+ * @param  {String} data Stringified JSON data.
39
+ */
40
+function saveAndExit(data) {
41
+    $.ajax({
42
+        type: 'POST',
43
+        url: '/saveExit',
44
+        data: data,
45
+        success: function(res) {
46
+            setTimeout(function() { alert(res); }, 3);
47
+        },
48
+        contentType: 'application/json;charset=UTF-8'
49
+    });
50
+}
51
+
52
+/**
53
+ * Collects and validates data from textarea.
54
+ * @returns  {String}  Stringified JSON data.
55
+ */
56
+function getSimpleData() {
57
+    var data = $("#yaml_data").val();
58
+    try {
59
+        var index = data.indexOf(changeStr)
60
+        if (index != -1) {
61
+            var lineNum = data.substring(0, index).split('\n').length;
62
+            alert('Please change value on line '+ lineNum + '!')
63
+            return null
64
+        }
65
+        data = jsyaml.load(data)
66
+    }
67
+    catch(err) {
68
+        alert(err)
69
+        return null
70
+    }
71
+    return JSON.stringify({yaml_data : data})
72
+}
73
+
74
+/**
75
+ * Function to save edit progress.
76
+ */
77
+function saveSimple() {
78
+    var data = getSimpleData()
79
+    if (data) {
80
+        save(data)
81
+    }
82
+}
83
+
84
+/**
85
+ * Function to save edit progress and shut down web server.
86
+ */
87
+function saveExitSimple() {
88
+    var data = getSimpleData()
89
+    if (data) {
90
+        saveAndExit(data)
91
+    }
92
+}

BIN
spyglass/utils/editor/static/favicon.ico View File


+ 68
- 0
spyglass/utils/editor/static/jquery-linedtextarea.css View File

@@ -0,0 +1,68 @@
1
+/**
2
+ * jQuery Lined Textarea Plugin
3
+ *   http://alan.blog-city.com/jquerylinedtextarea.htm
4
+ *
5
+ * Copyright (c) 2010 Alan Williamson
6
+ *
7
+ * Released under the MIT License:
8
+ * http://www.opensource.org/licenses/mit-license.php
9
+ *
10
+ * Usage:
11
+ *   Displays a line number count column to the left of the textarea
12
+ *
13
+ *   Class up your textarea with a given class, or target it directly
14
+ *   with JQuery Selectors
15
+ *
16
+ *   $(".lined").linedtextarea({
17
+ *   	selectedLine: 10,
18
+ *    selectedClass: 'lineselect'
19
+ *   });
20
+ *
21
+ */
22
+
23
+.linedwrap {
24
+	border: 1px solid #c0c0c0;
25
+	padding: 3px;
26
+}
27
+
28
+.linedtextarea {
29
+	padding: 0px;
30
+	margin: 0px;
31
+}
32
+
33
+.linedtextarea textarea, .linedwrap .codelines .lineno {
34
+	font-size: 10pt;
35
+	font-family: monospace;
36
+	line-height: normal !important;
37
+}
38
+
39
+.linedtextarea textarea {
40
+	padding-right:0.3em;
41
+	padding-top:0.3em;
42
+	border: 0;
43
+}
44
+
45
+.linedwrap .lines {
46
+	margin-top: 0px;
47
+	width: 50px;
48
+	float: left;
49
+	overflow: hidden;
50
+	border-right: 1px solid #c0c0c0;
51
+	margin-right: 10px;
52
+}
53
+
54
+.linedwrap .codelines {
55
+	padding-top: 5px;
56
+}
57
+
58
+.linedwrap .codelines .lineno {
59
+	color:#AAAAAA;
60
+	padding-right: 0.5em;
61
+	padding-top: 0.0em;
62
+	text-align: right;
63
+	white-space: nowrap;
64
+}
65
+
66
+.linedwrap .codelines .lineselect {
67
+	color: red;
68
+}

+ 126
- 0
spyglass/utils/editor/static/jquery-linedtextarea.js View File

@@ -0,0 +1,126 @@
1
+/**
2
+ * jQuery Lined Textarea Plugin
3
+ *   http://alan.blog-city.com/jquerylinedtextarea.htm
4
+ *
5
+ * Copyright (c) 2010 Alan Williamson
6
+ *
7
+ * Version:
8
+ *    $Id: jquery-linedtextarea.js 464 2010-01-08 10:36:33Z alan $
9
+ *
10
+ * Released under the MIT License:
11
+ *    http://www.opensource.org/licenses/mit-license.php
12
+ *
13
+ * Usage:
14
+ *   Displays a line number count column to the left of the textarea
15
+ *
16
+ *   Class up your textarea with a given class, or target it directly
17
+ *   with JQuery Selectors
18
+ *
19
+ *   $(".lined").linedtextarea({
20
+ *   	selectedLine: 10,
21
+ *    selectedClass: 'lineselect'
22
+ *   });
23
+ *
24
+ * History:
25
+ *   - 2010.01.08: Fixed a Google Chrome layout problem
26
+ *   - 2010.01.07: Refactored code for speed/readability; Fixed horizontal sizing
27
+ *   - 2010.01.06: Initial Release
28
+ *
29
+ */
30
+(function($) {
31
+
32
+	$.fn.linedtextarea = function(options) {
33
+
34
+		// Get the Options
35
+		var opts = $.extend({}, $.fn.linedtextarea.defaults, options);
36
+
37
+
38
+		/*
39
+		 * Helper function to make sure the line numbers are always
40
+		 * kept up to the current system
41
+		 */
42
+		var fillOutLines = function(codeLines, h, lineNo){
43
+			while ( (codeLines.height() - h ) <= 0 ){
44
+				if ( lineNo == opts.selectedLine )
45
+					codeLines.append("<div class='lineno lineselect'>" + lineNo + "</div>");
46
+				else
47
+					codeLines.append("<div class='lineno'>" + lineNo + "</div>");
48
+
49
+				lineNo++;
50
+			}
51
+			return lineNo;
52
+		};
53
+
54
+
55
+		/*
56
+		 * Iterate through each of the elements are to be applied to
57
+		 */
58
+		return this.each(function() {
59
+			var lineNo = 1;
60
+			var textarea = $(this);
61
+
62
+			/* Turn off the wrapping of as we don't want to screw up the line numbers */
63
+			textarea.attr("wrap", "off");
64
+			textarea.css({resize:'none'});
65
+			var originalTextAreaWidth	= textarea.outerWidth();
66
+
67
+			/* Wrap the text area in the elements we need */
68
+			textarea.wrap("<div class='linedtextarea'></div>");
69
+			var linedTextAreaDiv	= textarea.parent().wrap("<div class='linedwrap' style='width:" + originalTextAreaWidth + "px'></div>");
70
+			var linedWrapDiv 			= linedTextAreaDiv.parent();
71
+
72
+			linedWrapDiv.prepend("<div class='lines' style='width:50px'></div>");
73
+
74
+			var linesDiv	= linedWrapDiv.find(".lines");
75
+			linesDiv.height( textarea.height() + 6 );
76
+
77
+
78
+			/* Draw the number bar; filling it out where necessary */
79
+			linesDiv.append( "<div class='codelines'></div>" );
80
+			var codeLinesDiv	= linesDiv.find(".codelines");
81
+			lineNo = fillOutLines( codeLinesDiv, linesDiv.height(), 1 );
82
+
83
+			/* Move the textarea to the selected line */
84
+			if ( opts.selectedLine != -1 && !isNaN(opts.selectedLine) ){
85
+				var fontSize = parseInt( textarea.height() / (lineNo-2) );
86
+				var position = parseInt( fontSize * opts.selectedLine ) - (textarea.height()/2);
87
+				textarea[0].scrollTop = position;
88
+			}
89
+
90
+
91
+			/* Set the width */
92
+			var sidebarWidth					= linesDiv.outerWidth();
93
+			var paddingHorizontal 		= parseInt( linedWrapDiv.css("border-left-width") ) + parseInt( linedWrapDiv.css("border-right-width") ) + parseInt( linedWrapDiv.css("padding-left") ) + parseInt( linedWrapDiv.css("padding-right") );
94
+			var linedWrapDivNewWidth 	= originalTextAreaWidth - paddingHorizontal;
95
+			var textareaNewWidth			= originalTextAreaWidth - sidebarWidth - paddingHorizontal - 20;
96
+
97
+			textarea.width( textareaNewWidth );
98
+			linedWrapDiv.width( linedWrapDivNewWidth );
99
+
100
+
101
+
102
+			/* React to the scroll event */
103
+			textarea.scroll( function(tn){
104
+				var domTextArea		= $(this)[0];
105
+				var scrollTop 		= domTextArea.scrollTop;
106
+				var clientHeight 	= domTextArea.clientHeight;
107
+				codeLinesDiv.css( {'margin-top': (-1*scrollTop) + "px"} );
108
+				lineNo = fillOutLines( codeLinesDiv, scrollTop + clientHeight, lineNo );
109
+			});
110
+
111
+
112
+			/* Should the textarea get resized outside of our control */
113
+			textarea.resize( function(tn){
114
+				var domTextArea	= $(this)[0];
115
+				linesDiv.height( domTextArea.clientHeight + 6 );
116
+			});
117
+
118
+		});
119
+	};
120
+
121
+  // default options
122
+  $.fn.linedtextarea.defaults = {
123
+  	selectedLine: -1,
124
+  	selectedClass: 'lineselect'
125
+  };
126
+})(jQuery);

+ 1
- 0
spyglass/utils/editor/static/js-yaml.min.js
File diff suppressed because it is too large
View File


+ 38
- 0
spyglass/utils/editor/templates/yaml.html View File

@@ -0,0 +1,38 @@
1
+{% extends "bootstrap/base.html" %}
2
+{% block title %}YAML Editor{% endblock %}
3
+
4
+{% block styles %}
5
+{{super()}}
6
+    <link href="{{ url_for('static', filename='jquery-linedtextarea.css') }}" rel="stylesheet">
7
+{% endblock %}
8
+
9
+{% block scripts %}
10
+{{super()}}
11
+        <script src="{{ url_for('static', filename='js-yaml.min.js') }}"></script>
12
+        <script src="{{ url_for('static', filename='jquery-linedtextarea.js') }}"></script>
13
+        <script type="text/javascript">
14
+            var changeStr = '{{ change_str }}'
15
+            $(document).ready(function(){
16
+                $("#yaml_data").val(jsyaml.dump(JSON.parse('{{ data|safe }}')))
17
+                $("#yaml_data").linedtextarea();
18
+        });
19
+        </script>
20
+        <script src="{{ url_for('static', filename='app.js') }}"></script>
21
+{% endblock %}
22
+
23
+{% block content %}
24
+
25
+    <div class="container" style="margin-top:30px;">
26
+        <div class="form-group">
27
+            <pre>Edit your YAML (Update corresponding fields with {{ change_str }} text):</pre>
28
+            <div>
29
+                <textarea class="form-control linedtextarea" id='yaml_data' style="height: 500px;box-shadow: none;"></textarea>
30
+            </div>
31
+        </div>
32
+        <div class="form-group pull-right">
33
+            <button type="button" onclick="saveSimple()" class="btn btn-lg btn-success ">Save</button>
34
+            <button type="button" onclick="saveExitSimple()" class="btn btn-lg btn-primary ">Save and Exit</button>
35
+        </div>
36
+    </div>
37
+
38
+{% endblock %}

Loading…
Cancel
Save