From 36c8a8dc6027e751109d88536d4289a577b9a0bb Mon Sep 17 00:00:00 2001
From: Dmitrii Filippov <dmfilippov@google.com>
Date: Tue, 25 Feb 2020 16:49:08 +0100
Subject: [PATCH] Preserve html file formatting in links-updater.ts

Initial implementation of the links-updater.ts changes file formatting -
it doesn't preserve line breaks inside tags.
This change update links-updater.ts to avoid unnessecary formatting
change.

Change-Id: Ifccf588c63eefe69782e059fa63f3d1b70f570b9
---
 tools/node_tools/package.json                 |  4 +-
 .../links-updater.ts                          | 66 +++++++++++--------
 .../rollup.config.js                          |  9 ++-
 tools/node_tools/yarn.lock                    | 40 +++++++++++
 4 files changed, 89 insertions(+), 30 deletions(-)

diff --git a/tools/node_tools/package.json b/tools/node_tools/package.json
index c25d02e774..2af06b458d 100644
--- a/tools/node_tools/package.json
+++ b/tools/node_tools/package.json
@@ -3,12 +3,14 @@
   "description": "Gerrit Build Tools",
   "browser": false,
   "dependencies": {
-    "@types/parse5": "^4.0.0",
     "@bazel/rollup": "^0.41.0",
     "@bazel/typescript": "^1.0.1",
     "@types/node": "^10.17.12",
+    "@types/parse5": "^4.0.0",
+    "@types/parse5-html-rewriting-stream": "^5.1.2",
     "crisper": "^2.1.1",
     "dom5": "^3.0.1",
+    "parse5-html-rewriting-stream": "^5.1.1",
     "polymer-bundler": "^4.0.10",
     "polymer-cli": "^1.9.11",
     "rollup": "^1.27.5",
diff --git a/tools/node_tools/polygerrit_app_preprocessor/links-updater.ts b/tools/node_tools/polygerrit_app_preprocessor/links-updater.ts
index eb7212be50..24e445d2d5 100644
--- a/tools/node_tools/polygerrit_app_preprocessor/links-updater.ts
+++ b/tools/node_tools/polygerrit_app_preprocessor/links-updater.ts
@@ -16,12 +16,11 @@
  */
 
 import * as fs from "fs";
-import * as parse5 from "parse5";
+import RewritingStream from "parse5-html-rewriting-stream";
 import * as dom5 from "dom5";
 import {HtmlFileUtils, RedirectsResolver} from "./utils";
 import {Node} from 'dom5';
 import {readMultilineParamFile} from "../utils/command-line";
-import {FileUtils} from "../utils/file-utils";
 import { fail } from "../utils/common";
 import {JSONRedirects} from "./redirects";
 
@@ -34,9 +33,7 @@ import {JSONRedirects} from "./redirects";
  * Additionaly, update some test links (related to web-component-tester)
  */
 
-function main() {
-  console.log(process.cwd());
-
+async function main() {
   if (process.argv.length < 4) {
     console.info("Usage:\n\tnode links_updater.js input_output_param_files redirectFile.json\n");
     process.exit(1);
@@ -50,7 +47,7 @@ function main() {
   for(let i = 0; i < input.length; i += 2) {
     const srcFile = input[i];
     const targetFile = input[i + 1];
-    updater.updateFile(srcFile, targetFile);
+    await updater.updateFile(srcFile, targetFile);
   }
 }
 
@@ -66,29 +63,42 @@ class HtmlFileUpdater {
     isHtmlImport: (node: Node) => node.tagName === "link" && dom5.getAttribute(node, "rel") === "import" &&
         dom5.hasAttribute(node, "href")
   };
+
   public constructor(private readonly redirectsResolver: RedirectsResolver) {
   }
 
-  public updateFile(srcFile: string, targetFile: string) {
+  public async updateFile(srcFile: string, targetFile: string) {
     const html = fs.readFileSync(srcFile, "utf-8");
-    const ast = parse5.parseFragment(html, {locationInfo: true}) as Node;
-
-
-    const webComponentTesterImportNode = dom5.query(ast, HtmlFileUpdater.Predicates.isWebComponentTesterImport);
-    if(webComponentTesterImportNode) {
-      dom5.setAttribute(webComponentTesterImportNode,  "src", "/components/wct-browser-legacy/browser.js");
-    }
-
-    // Update all HTML imports
-    const updateHtmlImportHref = (htmlImportNode: Node) => this.updateRefAttribute(htmlImportNode, srcFile, "href");
-    dom5.queryAll(ast, HtmlFileUpdater.Predicates.isHtmlImport).forEach(updateHtmlImportHref);
-
-    // Update all <script src=...> tags
-    const updateScriptSrc = (scriptTagNode: Node) => this.updateRefAttribute(scriptTagNode, srcFile, "src");
-    dom5.queryAll(ast, HtmlFileUpdater.Predicates.isScriptWithSrcTag).forEach(updateScriptSrc);
-
-    const newContent = parse5.serialize(ast);
-    FileUtils.writeContent(targetFile, newContent);
+    const readStream = fs.createReadStream(srcFile, {encoding: "utf-8"});
+    const rewriterOutput = srcFile === targetFile ? targetFile + ".tmp" : targetFile;
+    const writeStream = fs.createWriteStream(rewriterOutput, {encoding: "utf-8"});
+    const rewriter = new RewritingStream();
+    (rewriter as any).tokenizer.preprocessor.bufferWaterline = Infinity;
+    rewriter.on("startTag", (tag: any) => {
+      if (HtmlFileUpdater.Predicates.isWebComponentTesterImport(tag)) {
+        dom5.setAttribute(tag, "src", "/components/wct-browser-legacy/browser.js");
+      } else if (HtmlFileUpdater.Predicates.isHtmlImport(tag)) {
+        this.updateRefAttribute(tag, srcFile, "href");
+      } else if (HtmlFileUpdater.Predicates.isScriptWithSrcTag(tag)) {
+        this.updateRefAttribute(tag, srcFile, "src");
+      } else {
+        const location = tag.sourceCodeLocation;
+        const raw = html.substring(location.startOffset, location.endOffset);
+        rewriter.emitRaw(raw);
+        return;
+      }
+      rewriter.emitStartTag(tag);
+    });
+    return new Promise<void>((resolve, reject) => {
+      writeStream.on("close", () => {
+        writeStream.close();
+        if (rewriterOutput !== targetFile) {
+          fs.renameSync(rewriterOutput, targetFile);
+        }
+        resolve();
+      });
+      readStream.pipe(rewriter).pipe(writeStream);
+    });
   }
 
   private getResolvedPath(parentHtml: string, href: string) {
@@ -109,14 +119,14 @@ class HtmlFileUpdater {
 
   private updateRefAttribute(node: Node, parentHtml: string, attributeName: string) {
     const ref = dom5.getAttribute(node, attributeName);
-    if(!ref) {
+    if (!ref) {
       fail(`Internal error - ${node} in ${parentHtml} doesn't have attribute ${attributeName}`);
     }
     const newRef = this.getResolvedPath(parentHtml, ref);
-    if(newRef === ref) {
+    if (newRef === ref) {
       return;
     }
-    dom5.setAttribute(node,  attributeName, newRef);
+    dom5.setAttribute(node, attributeName, newRef);
   }
 }
 
diff --git a/tools/node_tools/polygerrit_app_preprocessor/rollup.config.js b/tools/node_tools/polygerrit_app_preprocessor/rollup.config.js
index 1da44413ca..7109bde9fa 100644
--- a/tools/node_tools/polygerrit_app_preprocessor/rollup.config.js
+++ b/tools/node_tools/polygerrit_app_preprocessor/rollup.config.js
@@ -17,5 +17,12 @@
 
 
 export default {
-  external: ['fs', 'path', 'parse5', 'dom5']
+  external: ['fs', 'path', 'parse5', 'dom5', 'parse5-html-rewriting-stream'],
+  onwarn: warn => {
+    // Typescript adds helper methods for await and promise which look like
+    // var __awaiter = (this && this.__awaiter)
+    // It is safe to ignore this warning
+    if(warn.code === 'THIS_IS_UNDEFINED') { return; }
+    throw new Error(warn.message);
+  }
 };
diff --git a/tools/node_tools/yarn.lock b/tools/node_tools/yarn.lock
index eedea4ad02..0648c8d259 100644
--- a/tools/node_tools/yarn.lock
+++ b/tools/node_tools/yarn.lock
@@ -966,6 +966,26 @@
   dependencies:
     "@types/node" "*"
 
+"@types/parse5-html-rewriting-stream@^5.1.2":
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/@types/parse5-html-rewriting-stream/-/parse5-html-rewriting-stream-5.1.2.tgz#919d5bbf69ef61e11d873e7195891c3811491a03"
+  integrity sha512-7CHY6QlayurvYRST5xatE/ipIueph5V+EW2xU12P0CsNucuwygnuiE4foYsdQUEkhnKrTU62KmikANPnoxiGrg==
+  dependencies:
+    "@types/parse5-sax-parser" "*"
+
+"@types/parse5-sax-parser@*":
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/@types/parse5-sax-parser/-/parse5-sax-parser-5.0.1.tgz#f1e26e82bb09e48cb0c16ff6d1e88aea1e538fd5"
+  integrity sha512-wBEwg10aACLggnb44CwzAA27M1Jrc/8TR16zA61/rKO5XZoi7JSfLjdpXbshsm7wOlM6hpfvwygh40rzM2RsQQ==
+  dependencies:
+    "@types/node" "*"
+    "@types/parse5" "*"
+
+"@types/parse5@*":
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-5.0.2.tgz#a877a4658f8238c8266faef300ae41c84d72ec8a"
+  integrity sha512-BOl+6KDs4ItndUWUFchy3aEqGdHhw0BC4Uu+qoDonN/f0rbUnJbm71Ulj8Tt9jLFRaAxPLKvdS1bBLfx1qXR9g==
+
 "@types/parse5@^0.0.31":
   version "0.0.31"
   resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-0.0.31.tgz#e827a493a443b156e1b582a2e4c3bdc0040f2ee7"
@@ -5803,6 +5823,21 @@ parse-passwd@^1.0.0:
   resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
   integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=
 
+parse5-html-rewriting-stream@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/parse5-html-rewriting-stream/-/parse5-html-rewriting-stream-5.1.1.tgz#fc18570ba0d09b5091250956d1c3f716ef0a07b7"
+  integrity sha512-rbXBeMlJ3pk3tKxLKAUaqvQTZM5KTohXmZvYEv2gU9sQC70w65BxPsh3PVVnwiVNCnNYDtNZRqCKmiMlfdG07Q==
+  dependencies:
+    parse5 "^5.1.1"
+    parse5-sax-parser "^5.1.1"
+
+parse5-sax-parser@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/parse5-sax-parser/-/parse5-sax-parser-5.1.1.tgz#02834a9d08b23ea2d99584841c38be09d5247a15"
+  integrity sha512-9HIh6zd7bF1NJe95LPCUC311CekdOi55R+HWXNCsGY6053DWaMijVKOv1oPvdvPTvFicifZyimBVJ6/qvG039Q==
+  dependencies:
+    parse5 "^5.1.1"
+
 parse5@^1.4.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-1.5.1.tgz#9b7f3b0de32be78dc2401b17573ccaf0f6f59d94"
@@ -5813,6 +5848,11 @@ parse5@^4.0.0:
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608"
   integrity sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==
 
+parse5@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178"
+  integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==
+
 parseqs@0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/parseqs/-/parseqs-0.0.5.tgz#d5208a3738e46766e291ba2ea173684921a8b89d"