Log panel for arbitrary raw log messages

This change does the following:
1.  Break the circular reference between CTL and the webservice
2.  Establishes the log panel at the bottom of the screen
so that raw log messages can be show.
3.  Transfers the CTL logs to the UI by overwriting the CTL
logger.  This requires a new CTL client each transaction
4.  Adds in net/http specific logging for the backend

Change-Id: If7b01426c112669a11ffe8132f2cff59a4635db4

client/src/app/app.component.css

@@ -65,3 +65,28 @@
 .h3 {
   font-size: 12px;
 }
+
+.logs-action-buttons {
+  padding-bottom: 20px;
+}
+
+.logs-headers-align .mat-expansion-panel-header-title,
+.logs-headers-align .mat-expansion-panel-header-description {
+  flex-basis: 0;
+}
+
+.logs-headers-align .mat-expansion-panel-header-description {
+  justify-content: space-between;
+  align-items: center;
+}
+
+.logs-headers-align .mat-form-field + .mat-form-field {
+  margin-left: 8px;
+}
+
+.log-panel {
+  height: 20vh !important;
+  overflow: scroll;
+  display: flex;
+  flex-direction: column-reverse;
+}

client/src/app/app.component.html

@@ -61,6 +61,16 @@
       </mat-toolbar>
       <router-outlet></router-outlet>
       <span class="page-body"></span>
+      <mat-accordion class="logs-headers-align" style="display:none" id="logAccordion">
+        <mat-expansion-panel>
+          <mat-expansion-panel-header>
+            <mat-panel-title>
+              Logs
+            </mat-panel-title>
+          </mat-expansion-panel-header>
+          <div id="logPanel" class="log-panel"></div>
+        </mat-expansion-panel>
+      </mat-accordion>
       <mat-toolbar class="toolbar-footer">
         <h3>Airship UI &copy; {{ this.currentYear }}</h3>
         <span class="spacer"></span>

client/src/app/app.component.ts

@@ -1,4 +1,5 @@
-import { Component, OnInit } from '@angular/core';
+import { Component, OnInit, ViewChild } from '@angular/core';
+import { MatAccordion } from '@angular/material/expansion';
 import { environment } from 'src/environments/environment';
 import { IconService } from 'src/services/icon/icon.service';
 import { WebsocketService } from 'src/services/websocket/websocket.service';
@@ -15,6 +16,8 @@ import { AuthGuard } from 'src/services/auth-guard/auth-guard.service';
 })
 
 export class AppComponent implements OnInit, WSReceiver {
+  @ViewChild(MatAccordion) accordion: MatAccordion;
+
   className = this.constructor.name;
   type = 'ui';
   component = 'any';
@@ -52,11 +55,22 @@ export class AppComponent implements OnInit, WSReceiver {
     if (message.hasOwnProperty('error')) {
       this.websocketService.printIfToast(message);
     } else {
-      if (message.hasOwnProperty('dashboards')) {
+      switch (message.component) {
-        this.updateDashboards(message.dashboards);
+        case 'log':
-      } else {
+          Log.Debug(new LogMessage('Log message received in app', this.className, message));
-        // TODO (aschiefe): determine what should be notifications and what should be 86ed
+          const panel = document.getElementById('logPanel');
-        Log.Debug(new LogMessage('Message received in app', this.className, message));
+          panel.appendChild(document.createTextNode(message.message));
+          panel.appendChild(document.createElement('br'));
+          break;
+        case 'initialize':
+          Log.Debug(new LogMessage('Initialize message received in app', this.className, message));
+          if (message.hasOwnProperty('dashboards')) {
+            this.updateDashboards(message.dashboards);
+          }
+          break;
+        default:
+          Log.Debug(new LogMessage('Uncategorized message received in app', this.className, message));
+          break;
       }
     }
   }

client/src/services/auth-guard/auth-guard.service.spec.ts

@@ -1,7 +1,7 @@
 import { async, TestBed } from '@angular/core/testing';
 import { AuthGuard } from './auth-guard.service';
 import { RouterTestingModule } from '@angular/router/testing';
-import {ToastrModule} from 'ngx-toastr';
+import { ToastrModule } from 'ngx-toastr';
 
 describe('AuthGuardService', () => {
   let service: AuthGuard;

client/src/services/auth-guard/auth-guard.service.ts

@@ -1,9 +1,9 @@
-import { Injectable, OnDestroy } from '@angular/core';
+import { Injectable } from '@angular/core';
 import { Router, CanActivate, Event as RouterEvent, NavigationStart, NavigationEnd, NavigationCancel, NavigationError } from '@angular/router';
 import { Log } from 'src/services/log/log.service';
 import { LogMessage } from 'src/services/log/log-message';
 import { WebsocketService } from 'src/services/websocket/websocket.service';
-import { WSReceiver, WebsocketMessage, Authentication } from 'src/services/websocket/websocket.models';
+import { WSReceiver, WebsocketMessage } from 'src/services/websocket/websocket.models';
 
 @Injectable({
   providedIn: 'root'
@@ -28,10 +28,23 @@ export class AuthGuard implements WSReceiver, CanActivate {
     // blank out the local storage so we can't get re authenticate
     localStorage.removeItem('airshipUI-token');
 
+    // turn off the log panel, no logs for you!
+    AuthGuard.toggleLogPanel(false);
+
     // best to begin at the beginning so send the user back to /login
     this.router.navigate(['/login']);
   }
 
+  // flip the log panel according to where we are in the world
+  public static toggleLogPanel(authenticated): void {
+    const accordion = document.getElementById('logAccordion');
+    if (authenticated && accordion.style.display === 'none') {
+      accordion.style.display = '';
+    } else if (!authenticated) {
+      accordion.style.display = 'none';
+    }
+  }
+
   constructor(private websocketService: WebsocketService, private router: Router) {
     // create a static router so other components can access it if needs be
     AuthGuard.router = router;
@@ -101,6 +114,9 @@ export class AuthGuard implements WSReceiver, CanActivate {
     // flip the link if we're in or out of the fold
     this.toggleAuthButton(authenticated);
 
+    // flip the visibility of the log panel depending on the disposition of the user
+    AuthGuard.toggleLogPanel(authenticated);
+
     return authenticated;
   }
 
@@ -115,6 +131,8 @@ export class AuthGuard implements WSReceiver, CanActivate {
     }
   }
 
+
+
   // test the auth token to see if we can let the user see the page
   // TODO: maybe RBAC type of stuff may need to go here
   private isAuthenticated(): boolean {

go.sum

@@ -514,6 +514,7 @@ github.com/gophercloud/gophercloud v0.6.0/go.mod h1:GICNByuaEBibcjmjvI7QvYJSZEbG
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33 h1:893HsJqtxp9z1SF76gg6hY70hRY1wVlTSnC/h1yUDCo=
 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=

pkg/commands/root.go

@@ -22,6 +22,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"opendev.org/airship/airshipui/pkg/configs"
+	"opendev.org/airship/airshipui/pkg/ctl"
 	"opendev.org/airship/airshipui/pkg/log"
 	"opendev.org/airship/airshipui/pkg/webservice"
 )
@@ -68,6 +69,10 @@ func launch(cmd *cobra.Command, args []string) {
 		log.Fatalf("config %s", err)
 	}
 
+	// allows for the circular reference to the webservice package to be broken and allow for the sending
+	// of arbitrary messages from any package to the websocket
+	ctl.Init()
+
 	// start webservice and listen for the the ctl + c to exit
 	c := make(chan os.Signal)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)

pkg/configs/configs.go

@@ -96,6 +96,7 @@ const (
 	Baremetal    WsComponentType = "baremetal"
 	Document     WsComponentType = "document"
 	Auth         WsComponentType = "auth"
+	Log          WsComponentType = "log"
 
 	// auth sub components
 	Approved     WsSubComponentType = "approved"

pkg/ctl/airshipctl.go

@@ -16,7 +16,10 @@ package ctl
 
 import (
 	"opendev.org/airship/airshipctl/pkg/environment"
+	"opendev.org/airship/airshipctl/pkg/log"
 	"opendev.org/airship/airshipui/pkg/configs"
+	uiLog "opendev.org/airship/airshipui/pkg/log"
+	"opendev.org/airship/airshipui/pkg/webservice"
 )
 
 // CTLFunctionMap is a function map for the CTL functions that is referenced in the webservice
@@ -34,22 +37,72 @@ type Client struct {
 	settings *environment.AirshipCTLSettings
 }
 
-// NewClient initializes the airshipctl client for external usage.
+// LogInterceptor is just a struct to hold a pointer to the remote channel
-func NewClient() *Client {
+type LogInterceptor struct {
+	response configs.WsMessage
+}
+
+// Init allows for the circular reference to the webservice package to be broken and allow for the sending
+// of arbitrary messages from any package to the websocket
+func Init() {
+	webservice.AppendToFunctionMap(configs.CTL, map[configs.WsComponentType]func(configs.WsMessage) configs.WsMessage{
+		configs.Baremetal: HandleBaremetalRequest,
+		configs.Document:  HandleDocumentRequest,
+	})
+}
+
+// NewDefaultClient initializes the airshipctl client for external usage with default logging.
+func NewDefaultClient() *Client {
 	settings := &environment.AirshipCTLSettings{}
 	// ensure no error if airship config doesn't exist
 	settings.Create = true
 	settings.InitConfig()
 
-	c := &Client{
+	client := &Client{
 		settings: settings,
 	}
 
 	// set verbosity to true
-	c.settings.Debug = true
+	client.settings.Debug = true
 
-	return c
+	return client
 }
 
-// initialize the connection to airshipctl
+// NewClient initializes the airshipctl client for external usage with the logging overridden.
-var c *Client = NewClient()
+func NewClient(request configs.WsMessage) *Client {
+	client := NewDefaultClient()
+
+	// init the interceptor to send messages to the UI
+	// TODO: Unsure how this will be handled with overlapping runs
+	log.Init(client.settings.Debug, NewLogInterceptor(request))
+
+	return client
+}
+
+// NewLogInterceptor will construct a channel writer for use with the logger
+func NewLogInterceptor(request configs.WsMessage) *LogInterceptor {
+	// TODO: determine if we're only getting stub responses and if we don't have to pick things out that we care about
+	// This is a stub response used by the writer to kick out messages to the UI
+	response := configs.WsMessage{
+		Type:      configs.UI,
+		Component: configs.Log,
+		SessionID: request.SessionID,
+	}
+
+	return &LogInterceptor{
+		response: response,
+	}
+}
+
+// Write satisfies the implementation of io.Writer.
+// The intention is to hijack the log output for a progress bar on the UI
+func (cw *LogInterceptor) Write(data []byte) (n int, err error) {
+	response := cw.response
+	response.Message = string(data)
+	if err = webservice.WebSocketSend(response); err != nil {
+		uiLog.Errorf("Error receiving / sending message: %s\n", err)
+		return len(data), err
+	}
+
+	return len(data), nil
+}

pkg/ctl/baremetal.go

@@ -36,8 +36,10 @@ func HandleBaremetalRequest(request configs.WsMessage) configs.WsMessage {
 	switch subComponent {
 	case configs.GenerateISO:
 		// since this is long running cache it up
+		// TODO: Test before running the geniso
 		runningRequests[subComponent] = true
-		message, err = c.generateIso()
+		client := NewClient(request)
+		message, err = client.generateIso()
 		// now that we're done forget we did anything
 		delete(runningRequests, subComponent)
 	default:

pkg/ctl/document.go

@@ -41,26 +41,29 @@ func HandleDocumentRequest(request configs.WsMessage) configs.WsMessage {
 	var err error
 	var message string
 	var id string
+
+	client := NewClient(request)
+
 	switch request.SubComponent {
 	case configs.DocPull:
-		message, err = c.docPull()
+		message, err = client.docPull()
 	case configs.YamlWrite:
 		id = request.ID
-		response.Name, response.YAML, err = writeYamlFile(id, request.YAML)
+		response.Name, response.YAML, err = client.writeYamlFile(id, request.YAML)
 		message = fmt.Sprintf("File '%s' saved successfully", response.Name)
 	case configs.GetYaml:
 		id = request.ID
-		response.Name, response.YAML, err = getYaml(id)
+		response.Name, response.YAML, err = client.getYaml(id)
 	case configs.GetPhaseTree:
-		response.Data, err = GetPhaseTree()
+		response.Data, err = client.GetPhaseTree()
 	case configs.GetPhaseDocuments:
 		id = request.ID
 		response.Data, err = GetPhaseDocuments(request.ID)
 	case configs.GetPhaseSourceFiles:
 		id = request.ID
-		response.Data, err = GetPhaseSourceFiles(request.ID)
+		response.Data, err = client.GetPhaseSourceFiles(request.ID)
 	case configs.GetTarget:
-		message = getTarget()
+		message = client.getTarget()
 	default:
 		err = fmt.Errorf("Subcomponent %s not found", request.SubComponent)
 	}
@@ -75,7 +78,7 @@ func HandleDocumentRequest(request configs.WsMessage) configs.WsMessage {
 	return response
 }
 
-func getTarget() string {
+func (c *Client) getTarget() string {
 	m, err := c.settings.Config.CurrentContextManifest()
 	if err != nil {
 		return "unknown"
@@ -84,11 +87,11 @@ func getTarget() string {
 	return filepath.Join(m.TargetPath, m.SubPath)
 }
 
-func getYaml(id string) (string, string, error) {
+func (c *Client) getYaml(id string) (string, string, error) {
 	obj := index[id]
 	switch t := obj.(type) {
 	case string:
-		return getFileYaml(t)
+		return c.getFileYaml(t)
 	case document.Document:
 		return getDocumentYaml(t)
 	default:
@@ -106,7 +109,7 @@ func getDocumentYaml(doc document.Document) (string, string, error) {
 	return title, base64.StdEncoding.EncodeToString(bytes), nil
 }
 
-func getFileYaml(path string) (string, string, error) {
+func (c *Client) getFileYaml(path string) (string, string, error) {
 	ccm, err := c.settings.Config.CurrentContextManifest()
 	if err != nil {
 		return "", "", err
@@ -139,7 +142,7 @@ func getFileYaml(path string) (string, string, error) {
 	return title, base64.StdEncoding.EncodeToString(bytes), nil
 }
 
-func writeYamlFile(id, yaml64 string) (string, string, error) {
+func (c *Client) writeYamlFile(id, yaml64 string) (string, string, error) {
 	path, ok := index[id].(string)
 	if !ok {
 		return "", "", fmt.Errorf("ID %s not found", id)
@@ -155,7 +158,7 @@ func writeYamlFile(id, yaml64 string) (string, string, error) {
 		return "", "", err
 	}
 
-	return getFileYaml(path)
+	return c.getFileYaml(path)
 }
 
 func (c *Client) docPull() (string, error) {

pkg/ctl/tree.go

@@ -40,16 +40,21 @@ type PhaseObj struct {
 }
 
 func buildPhaseIndex() map[string]PhaseObj {
+	client := NewDefaultClient()
+	return client.buildPhaseIndex()
+}
+
+func (client *Client) buildPhaseIndex() map[string]PhaseObj {
 	idx := map[string]PhaseObj{}
 
 	// get target path from ctl settings
-	tp, err := c.settings.Config.CurrentContextTargetPath()
+	tp, err := client.settings.Config.CurrentContextTargetPath()
 	if err != nil {
 		log.Errorf("Error building phase index: %s", err)
 		return nil
 	}
 
-	cmd := phase.Cmd{AirshipCTLSettings: c.settings}
+	cmd := phase.Cmd{AirshipCTLSettings: client.settings}
 
 	plan, err := cmd.Plan()
 	if err != nil {
@@ -81,41 +86,39 @@ func buildPhaseIndex() map[string]PhaseObj {
 // GetPhaseTree builds the initial structure of the phase tree
 // consisting of phase Groups and Phases. Individual phase source
 // files or rendered documents will be lazy loaded as needed
-func GetPhaseTree() ([]KustomNode, error) {
+func (client *Client) GetPhaseTree() ([]KustomNode, error) {
 	nodes := []KustomNode{}
 
 	grpMap := map[string][]KustomNode{}
-
+	for id, po := range phaseIndex {
-	if phaseIndex != nil {
+		pNode := KustomNode{
-		for id, po := range phaseIndex {
+			ID:          id,
-			pNode := KustomNode{
+			Name:        fmt.Sprintf("Phase: %s", po.Name),
-				ID:          id,
+			IsPhaseNode: true,
-				Name:        fmt.Sprintf("Phase: %s", po.Name),
-				IsPhaseNode: true,
-			}
-
-			children, err := GetPhaseSourceFiles(id)
-			if err != nil {
-				// TODO(mfuller): push an error to UI so it can be handled by
-				// toastr service, pending refactor of webservice and configs pkgs
-				log.Errorf("Error building tree for phase '%s': %s", po.Name, err)
-				pNode.HasError = true
-			} else {
-				pNode.Children = children
-			}
-
-			grpMap[po.Group] = append(grpMap[po.Group], pNode)
 		}
 
-		for name, phases := range grpMap {
+		children, err := client.GetPhaseSourceFiles(id)
-			gNode := KustomNode{
+		if err != nil {
-				ID:       uuid.New().String(),
+			// TODO(mfuller): push an error to UI so it can be handled by
-				Name:     fmt.Sprintf("Group: %s", name),
+			// toastr service, pending refactor of webservice and configs pkgs
-				Children: phases,
+			log.Errorf("Error building tree for phase '%s': %s", po.Name, err)
-			}
+			pNode.HasError = true
-			nodes = append(nodes, gNode)
+		} else {
+			pNode.Children = children
 		}
+
+		grpMap[po.Group] = append(grpMap[po.Group], pNode)
 	}
+
+	for name, phases := range grpMap {
+		gNode := KustomNode{
+			ID:       uuid.New().String(),
+			Name:     fmt.Sprintf("Group: %s", name),
+			Children: phases,
+		}
+		nodes = append(nodes, gNode)
+	}
+
 	return nodes, nil
 }
 
@@ -201,7 +204,7 @@ func sortDocuments(path string) (map[string]map[string][]document.Document, erro
 // all of the directories that will be traversed when kustomize
 // builds the document bundle. The tree hierarchy is:
 // kustomize "type" (like function) -> directory name -> file name
-func GetPhaseSourceFiles(id string) ([]KustomNode, error) {
+func (client *Client) GetPhaseSourceFiles(id string) ([]KustomNode, error) {
 	if index == nil {
 		index = map[string]interface{}{}
 	}
@@ -213,7 +216,7 @@ func GetPhaseSourceFiles(id string) ([]KustomNode, error) {
 			return nil, err
 		}
 
-		dm, err := createDirsMap(dirs)
+		dm, err := client.createDirsMap(dirs)
 		if err != nil {
 			return nil, err
 		}
@@ -326,10 +329,10 @@ func getKustomizeDirs(entrypoint string) ([]string, error) {
 }
 
 // helper function to group kustomize dirs by type (i.e. function, composite, etc)
-func createDirsMap(dirs []string) (map[string][][]string, error) {
+func (client *Client) createDirsMap(dirs []string) (map[string][][]string, error) {
 	dm := map[string][][]string{}
 
-	tp, err := c.settings.Config.CurrentContextTargetPath()
+	tp, err := client.settings.Config.CurrentContextTargetPath()
 	if err != nil {
 		return nil, err
 	}

pkg/log/log.go

@@ -110,6 +110,11 @@ func Writer() io.Writer {
 	return airshipLog.Writer()
 }
 
+// Logger is used by things like net/http to overwrite their standard logging
+func Logger() *log.Logger {
+	return airshipLog
+}
+
 func writeLog(level int, v ...interface{}) {
 	// determine if we need to display the logs
 	if level <= LogLevel {

pkg/webservice/server.go

@@ -15,6 +15,7 @@
 package webservice
 
 import (
+	"crypto/tls"
 	"net/http"
 	"strconv"
 
@@ -54,6 +55,17 @@ func serveFile(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// getCertificates returns the cert chain in a way that the net/http server struct expects
+func getCertificates() []tls.Certificate {
+	cert, err := tls.LoadX509KeyPair(configs.UIConfig.WebService.PublicKey, configs.UIConfig.WebService.PrivateKey)
+	if err != nil {
+		log.Fatal("Unable to load certificates, check the definition in etc/airshipui.json")
+	}
+	var certSlice []tls.Certificate
+	certSlice = append(certSlice, cert)
+	return certSlice
+}
+
 // WebServer will run the handler functions for WebSockets
 func WebServer() {
 	webServerMux := http.NewServeMux()
@@ -72,8 +84,19 @@ func WebServer() {
 	// Calculate the address and start on the host and port specified in the config
 	addr := configs.UIConfig.WebService.Host + ":" + strconv.Itoa(configs.UIConfig.WebService.Port)
 	log.Infof("Attempting to start webservice on %s", addr)
-	log.Fatal(http.ListenAndServeTLS(addr,
+
-		configs.UIConfig.WebService.PublicKey,
+	// configure logging & TLS for the http server
-		configs.UIConfig.WebService.PrivateKey,
+	server := &http.Server{
-		webServerMux))
+		Addr: addr,
+		TLSConfig: &tls.Config{
+			InsecureSkipVerify: false,
+			ServerName:         configs.UIConfig.WebService.Host,
+			Certificates:       getCertificates(),
+		},
+		Handler:  webServerMux,
+		ErrorLog: log.Logger(),
+	}
+
+	// kick off the server, and good luck
+	log.Fatal(server.ListenAndServeTLS("", ""))
 }

pkg/webservice/websocket.go

@@ -24,11 +24,10 @@ import (
 	"github.com/google/uuid"
 	"github.com/gorilla/websocket"
 	"opendev.org/airship/airshipui/pkg/configs"
-	"opendev.org/airship/airshipui/pkg/ctl"
 	"opendev.org/airship/airshipui/pkg/log"
 )
 
-// session is a struct to hold information about a given session
+// Session is a struct to hold information about a given session
 type session struct {
 	id         string
 	jwt        string
@@ -52,7 +51,14 @@ var functionMap = map[configs.WsRequestType]map[configs.WsComponentType]func(con
 		configs.Keepalive: keepaliveReply,
 		configs.Auth:      handleAuth,
 	},
-	configs.CTL: ctl.CTLFunctionMap,
+}
+
+// AppendToFunctionMap allows us to break up the circular reference from the other packages
+// It does however require them to implement an init function to append them
+// TODO: maybe some form of an interface to enforce this may be necessary?
+func AppendToFunctionMap(requestType configs.WsRequestType,
+	functions map[configs.WsComponentType]func(configs.WsMessage) configs.WsMessage) {
+	functionMap[requestType] = functions
 }
 
 // handle the origin request & upgrade to websocket