// Copyright (C) 2015 The Android Open Source Project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "archive/zip" "bufio" "bytes" "compress/gzip" "encoding/json" "errors" "flag" "io" "io/ioutil" "log" "net" "net/http" "net/url" "os" "os/exec" "path/filepath" "regexp" "strings" "sync" "time" "golang.org/x/tools/godoc/vfs/httpfs" "golang.org/x/tools/godoc/vfs/zipfs" ) var ( plugins = flag.String("plugins", "", "comma seperated plugin paths to serve") port = flag.String("port", "localhost:8081", "address to serve HTTP requests on") host = flag.String("host", "gerrit-review.googlesource.com", "Host to proxy requests to") scheme = flag.String("scheme", "https", "URL scheme") cdnPattern = regexp.MustCompile("https://cdn.googlesource.com/polygerrit_ui/[0-9.]*") bundledPluginsPattern = regexp.MustCompile("https://cdn.googlesource.com/polygerrit_assets/[0-9.]*") ) func main() { flag.Parse() fontsArchive, err := openDataArchive("fonts.zip") if err != nil { log.Fatal(err) } workspace := os.Getenv("BUILD_WORKSPACE_DIRECTORY") if err := os.Chdir(filepath.Join(workspace, "polygerrit-ui")); err != nil { log.Fatal(err) } compiledSrcPath := filepath.Join(workspace, "./.ts-out/server-go") tsInstance := newTypescriptInstance( filepath.Join(workspace, "./node_modules/.bin/tsc"), filepath.Join(workspace, "./polygerrit-ui/app/tsconfig.json"), compiledSrcPath, ) if err := tsInstance.StartWatch(); err != nil { log.Fatal(err) } dirListingMux := http.NewServeMux() dirListingMux.Handle("/styles/", http.StripPrefix("/styles/", http.FileServer(http.Dir("app/styles")))) dirListingMux.Handle("/samples/", http.StripPrefix("/samples/", http.FileServer(http.Dir("app/samples")))) dirListingMux.Handle("/elements/", http.StripPrefix("/elements/", http.FileServer(http.Dir("app/elements")))) dirListingMux.Handle("/behaviors/", http.StripPrefix("/behaviors/", http.FileServer(http.Dir("app/behaviors")))) http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { // If typescript compiler hasn't finished yet, wait for it tsInstance.WaitForCompilationComplete() handleSrcRequest(compiledSrcPath, dirListingMux, w, req) }) http.Handle("/fonts/", addDevHeadersMiddleware(http.FileServer(httpfs.New(zipfs.New(fontsArchive, "fonts"))))) http.HandleFunc("/index.html", handleIndex) http.HandleFunc("/changes/", handleProxy) http.HandleFunc("/accounts/", handleProxy) http.HandleFunc("/config/", handleProxy) http.HandleFunc("/projects/", handleProxy) http.HandleFunc("/static/", handleProxy) http.HandleFunc("/accounts/self/detail", handleAccountDetail) if len(*plugins) > 0 { http.Handle("/plugins/", http.StripPrefix("/plugins/", http.FileServer(http.Dir("../plugins")))) log.Println("Local plugins from", "../plugins") } else { http.HandleFunc("/plugins/", handleProxy) // Serve local plugins from `plugins_` http.Handle("/plugins_/", http.StripPrefix("/plugins_/", http.FileServer(http.Dir("../plugins")))) } log.Println("Serving on port", *port) log.Fatal(http.ListenAndServe(*port, &server{})) } func addDevHeadersMiddleware(h http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, req *http.Request) { addDevHeaders(writer) h.ServeHTTP(writer, req) }) } func addDevHeaders(writer http.ResponseWriter) { writer.Header().Set("Access-Control-Allow-Origin", "*") writer.Header().Set("Cache-Control", "public, max-age=10, must-revalidate") } func handleSrcRequest(compiledSrcPath string, dirListingMux *http.ServeMux, writer http.ResponseWriter, originalRequest *http.Request) { parsedUrl, err := url.Parse(originalRequest.RequestURI) if err != nil { writer.WriteHeader(500) return } if parsedUrl.Path != "/" && strings.HasSuffix(parsedUrl.Path, "/") { dirListingMux.ServeHTTP(writer, originalRequest) return } normalizedContentPath := parsedUrl.Path if !strings.HasPrefix(normalizedContentPath, "/") { normalizedContentPath = "/" + normalizedContentPath } isJsFile := strings.HasSuffix(normalizedContentPath, ".js") || strings.HasSuffix(normalizedContentPath, ".mjs") isTsFile := strings.HasSuffix(normalizedContentPath, ".ts") // Source map in a compiled js file point to a file inside /app/... directory // Browser tries to load original file from the directory when debugger is // activated. In this case we return original content without any processing isOriginalFileRequest := strings.HasPrefix(normalizedContentPath, "/polygerrit-ui/app/") && (isTsFile || isJsFile) data, err := getContent(compiledSrcPath, normalizedContentPath, isOriginalFileRequest) if err != nil { if !isOriginalFileRequest { data, err = getContent(compiledSrcPath, normalizedContentPath+".js", false) } if err != nil { writer.WriteHeader(404) return } isJsFile = true } if isOriginalFileRequest { // Explicitly set text/html Content-Type. If live code tries // to import javascript from the /app/ folder accidentally, browser fails // with the import error, so we can catch this problem easily. writer.Header().Set("Content-Type", "text/html") } else if isJsFile { // import ... from '@polymer/decorators' // must be transformed into // import ... from '@polymer/decorators/lib/decorators.js' // The correct way to do it is to use value of the "main" property // from the @polymer/decorators/package.json. However, parsing package.json // is overcomplicated right now, hard-code exact path here. moduleImportRegexp := regexp.MustCompile("(?m)^(import.*)'@polymer/decorators';$") data = moduleImportRegexp.ReplaceAll(data, []byte("$1 '@polymer/decorators/lib/decorators.js';")) // The following code updates import statements. // 1. if an in imported file has .js or .mjs extension, the code keeps // the file extension unchanged. Otherwise, it adds .js extension // 2. For module imports it adds '/node_modules/' prefix. // Examples: // '@polymer/polymer.js' -> '/node_modules/@polymer/polymer.js' // 'page/page.mjs' -> '/node_modules/page.mjs' // '@polymer/iron-icon' -> '/node_modules/@polymer/iron-icon.js' // './element/file' -> './element/file.js' moduleImportRegexp = regexp.MustCompile(`(?m)^(import.*|export.* from )['"](.*?)(\.(m?)js)?['"];$`) data = moduleImportRegexp.ReplaceAll(data, []byte("$1'$2.${4}js';")) moduleImportRegexp = regexp.MustCompile(`(?m)^(import.*|export.* from )['"]([^/.].*)['"];$`) data = moduleImportRegexp.ReplaceAll(data, []byte("$1'/node_modules/$2';")) // The es module version of rxjs can be found in the _esm2015/ directory. moduleImportRegexp = regexp.MustCompile("(?m)^((import|export).*'/node_modules/rxjs)(.*).js(';)$") data = moduleImportRegexp.ReplaceAll(data, []byte("$1/_esm2015$3/index.js$4")) // The es module version of tslib.js can be found in tslib.es6.js. moduleImportRegexp = regexp.MustCompile("(?m)^((import|export).*'/node_modules/)tslib.js';$") data = moduleImportRegexp.ReplaceAll(data, []byte("${1}tslib/tslib.es6.js';")) // 'lit-element' imports and exports have to be resolved to 'lit-element/lit-element.js'. moduleImportRegexp = regexp.MustCompile("(?m)^((import|export).*'/node_modules/)lit-(element|html).js';$") data = moduleImportRegexp.ReplaceAll(data, []byte("${1}lit-${3}/lit-${3}.js';")) if strings.HasSuffix(normalizedContentPath, "/node_modules/page/page.js") { // Can't import page.js directly, because this is undefined. // Replace it with window // The same replace exists in karma.conf.js // Rollup makes this replacement automatically pageJsRegexp := regexp.MustCompile(`(?m)^}\(this, \(function \(\) { 'use strict';$`) newData := pageJsRegexp.ReplaceAll(data, []byte("}(window, (function () { 'use strict';")) if len(newData) == len(data) { log.Fatal("The page.js was updated. Please update regexp/replace accordingly") } data = newData } writer.Header().Set("Content-Type", "application/javascript") } else if strings.HasSuffix(normalizedContentPath, ".css") { writer.Header().Set("Content-Type", "text/css") } else if strings.HasSuffix(normalizedContentPath, "_test.html") { moduleImportRegexp := regexp.MustCompile("(?m)^(import.*)'([^/.].*)';$") data = moduleImportRegexp.ReplaceAll(data, []byte("$1 '/node_modules/$2';")) writer.Header().Set("Content-Type", "text/html") } else if strings.HasSuffix(normalizedContentPath, ".html") { writer.Header().Set("Content-Type", "text/html") } writer.WriteHeader(200) addDevHeaders(writer) writer.Write(data) } func getContent(compiledSrcPath string, normalizedContentPath string, isOriginalFileRequest bool) ([]byte, error) { // normalizedContentPath must always starts with '/' if isOriginalFileRequest { data, err := ioutil.ReadFile(normalizedContentPath[len("/polygerrit-ui/"):]) if err != nil { return nil, errors.New("File not found") } return data, nil } // gerrit loads gr-app.js as an ordinary script, without type="module" attribute. // If server.go serves this file as is, browser shows the error: // Uncaught SyntaxError: Cannot use import statement outside a module // // To load non-bundled gr-app.js as a module, we "virtually" renames original // gr-app.js to gr-app.mjs and load it with dynamic import. // // Another option is to patch rewriteHostPage function and add type="module" attribute // to snippet that also // contains window.INITIAL_DATA=... // Here we rely on the fact that the ") builder := new(strings.Builder) builder.WriteString( "window.INITIAL_DATA['/config/server/info'].plugin.html_resource_paths = []; ") builder.WriteString( "window.INITIAL_DATA['/config/server/info'].plugin.js_resource_paths = []; ") for _, p := range strings.Split(*plugins, ",") { if filepath.Ext(p) == ".html" { builder.WriteString( "window.INITIAL_DATA['/config/server/info'].plugin.html_resource_paths.push('" + p + "'); ") } if filepath.Ext(p) == ".js" { builder.WriteString( "window.INITIAL_DATA['/config/server/info'].plugin.js_resource_paths.push('" + p + "'); ") } } replaced = replaced[:insertionPoint] + builder.String() + replaced[insertionPoint:] } return strings.NewReader(replaced) } func injectLocalPlugins(reader io.Reader) io.Reader { if len(*plugins) == 0 { return reader } // Skip escape prefix io.CopyN(ioutil.Discard, reader, 5) dec := json.NewDecoder(reader) var response map[string]interface{} err := dec.Decode(&response) if err != nil { log.Fatal(err) } // Configuration path in the JSON server response jsPluginsPath := []string{"plugin", "js_resource_paths"} htmlPluginsPath := []string{"plugin", "html_resource_paths"} htmlResources := getJsonPropByPath(response, htmlPluginsPath).([]interface{}) jsResources := getJsonPropByPath(response, jsPluginsPath).([]interface{}) for _, p := range strings.Split(*plugins, ",") { if filepath.Ext(p) == ".html" { htmlResources = append(htmlResources, p) } if filepath.Ext(p) == ".js" { jsResources = append(jsResources, p) } } setJsonPropByPath(response, jsPluginsPath, jsResources) setJsonPropByPath(response, htmlPluginsPath, htmlResources) reader, writer := io.Pipe() go func() { defer writer.Close() io.WriteString(writer, ")]}'") // Write escape prefix err := json.NewEncoder(writer).Encode(&response) if err != nil { log.Fatal(err) } }() return reader } func handleAccountDetail(w http.ResponseWriter, r *http.Request) { http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) } type gzipResponseWriter struct { io.WriteCloser http.ResponseWriter } func newGzipResponseWriter(w http.ResponseWriter) *gzipResponseWriter { gz := gzip.NewWriter(w) return &gzipResponseWriter{WriteCloser: gz, ResponseWriter: w} } func (w gzipResponseWriter) Write(b []byte) (int, error) { return w.WriteCloser.Write(b) } func (w gzipResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { h, ok := w.ResponseWriter.(http.Hijacker) if !ok { return nil, nil, errors.New("gzipResponseWriter: ResponseWriter does not satisfy http.Hijacker interface") } return h.Hijack() } type server struct{} // Any path prefixes that should resolve to index.html. var ( fePaths = []string{"/q/", "/c/", "/id/", "/p/", "/x/", "/dashboard/", "/admin/", "/settings/"} issueNumRE = regexp.MustCompile(`^\/\d+\/?$`) ) func (_ *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.Printf("%s %s %s %s\n", r.Proto, r.Method, r.RemoteAddr, r.URL) for _, prefix := range fePaths { if strings.HasPrefix(r.URL.Path, prefix) || r.URL.Path == "/" { r.URL.Path = "/index.html" log.Println("Redirecting to /index.html") break } else if match := issueNumRE.Find([]byte(r.URL.Path)); match != nil { r.URL.Path = "/index.html" log.Println("Redirecting to /index.html") break } } if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { http.DefaultServeMux.ServeHTTP(w, r) return } w.Header().Set("Content-Encoding", "gzip") addDevHeaders(w) gzw := newGzipResponseWriter(w) defer gzw.Close() http.DefaultServeMux.ServeHTTP(gzw, r) } // Typescript compiler support // The code below runs typescript compiler in watch mode and redirect // all output from the compiler to the standard logger with the prefix "TSC -" // Additionally, the code analyzes messages produced by the typescript compiler // and allows to wait until compilation is finished. var ( tsStartingCompilation = "- Starting compilation in watch mode..." tsFileChangeDetectedMsg = "- File change detected. Starting incremental compilation..." // If there is only one error typescript outputs: // Found 1 error // In all other cases it outputs // Found X errors tsStartWatchingMsg = regexp.MustCompile(`^.* - Found \d+ error(s)?\. Watching for file changes\.$`) waitForNextChangeInterval = 1 * time.Second ) // typescriptLogWriter implements Writer interface and receives output // (stdout and stderr) from the typescript compiler. It reads incoming // data line-by-line, analyzes each line and updates compilationDoneWaiter // according to the current compiler state. Additionally, the // typescriptLogWriter passes all incoming lines to the underlying logger. type typescriptLogWriter struct { // unfinishedLine stores the portion of line which was partially received // (i.e. all text received after the last EOL (\n) mark. unfinishedLine string // logger is used to pass-through all received strings logger *log.Logger // when WaitGroup counter is 0 the compilation is complete compilationDoneWaiter *sync.WaitGroup } func newTypescriptLogWriter(compilationCompleteWaiter *sync.WaitGroup) *typescriptLogWriter { return &typescriptLogWriter{ logger: log.New(log.Writer(), "TSC - ", log.Flags()), compilationDoneWaiter: compilationCompleteWaiter, } } func (lw typescriptLogWriter) Write(p []byte) (n int, err error) { // The input p can contain several lines and/or the partial line // Code splits the input by EOL marker (\n) and stores the unfinished line // for the next call to Write. partialText := lw.unfinishedLine + string(p) lines := strings.Split(partialText, "\n") fullLines := lines if strings.HasSuffix(partialText, "\n") { lw.unfinishedLine = "" } else { fullLines = lines[:len(lines)-1] lw.unfinishedLine = lines[len(lines)-1] } for _, fullLine := range fullLines { text := strings.TrimSpace(fullLine) if text == "" { continue } if strings.HasSuffix(text, tsFileChangeDetectedMsg) || strings.HasSuffix(text, tsStartingCompilation) { lw.compilationDoneWaiter.Add(1) } if tsStartWatchingMsg.MatchString(text) { // A source code can be changed while previous compiler run is in progress. // In this case typescript reruns compilation again almost immediately // after the previous run finishes. To detect this situation, we are // waiting waitForNextChangeInterval before decreasing the counter. // If another compiler run is started in this interval, we will wait // again until it finishes. go func() { time.Sleep(waitForNextChangeInterval) lw.compilationDoneWaiter.Done() }() } lw.logger.Print(text) } return len(p), nil } type typescriptInstance struct { cmd *exec.Cmd compilationCompleteWaiter *sync.WaitGroup } func newTypescriptInstance(tscBinaryPath string, projectPath string, outdir string) *typescriptInstance { cmd := exec.Command(tscBinaryPath, "--watch", "--preserveWatchOutput", "--project", projectPath, "--outDir", outdir) compilationCompleteWaiter := &sync.WaitGroup{} logWriter := newTypescriptLogWriter(compilationCompleteWaiter) // Note 1: (from https://golang.org/pkg/os/exec/#Cmd) // If Stdout and Stderr are the same writer, and have a type that can // be compared with ==, at most one goroutine at a time will call Write. // // Note 2: The typescript compiler reports all compilation errors to // stdout by design (see https://github.com/microsoft/TypeScript/issues/615) // It writes to stderr only when something unexpected happens (like internal // exceptions). To print such errors in the same way as standard typescript // error, the same logWriter is used both for Stdout and Stderr. // // If Stderr arrives in the middle of ordinary typescript output (i.e. // something unexpected happens), the server.go can stop respond to http // requests. However, this is not a problem for us: typescript compiler and // server.go must be restarted anyway. cmd.Stdout = logWriter cmd.Stderr = logWriter return &typescriptInstance{ cmd: cmd, compilationCompleteWaiter: compilationCompleteWaiter, } } func (ts *typescriptInstance) StartWatch() error { err := ts.cmd.Start() if err != nil { return err } go func() { ts.cmd.Wait() log.Fatal("Typescript exits unexpected") }() return nil } func (ts *typescriptInstance) WaitForCompilationComplete() { ts.compilationCompleteWaiter.Wait() }