Merge "Preserve html file formatting in links-updater.ts"

tools/node_tools/package.json

@@ -3,12 +3,14 @@
   "description": "Gerrit Build Tools",
   "browser": false,
   "dependencies": {
-    "@types/parse5": "^4.0.0",
     "@bazel/rollup": "^0.41.0",
     "@bazel/typescript": "^1.0.1",
     "@types/node": "^10.17.12",
+    "@types/parse5": "^4.0.0",
+    "@types/parse5-html-rewriting-stream": "^5.1.2",
     "crisper": "^2.1.1",
     "dom5": "^3.0.1",
+    "parse5-html-rewriting-stream": "^5.1.1",
     "polymer-bundler": "^4.0.10",
     "polymer-cli": "^1.9.11",
     "rollup": "^1.27.5",

tools/node_tools/polygerrit_app_preprocessor/links-updater.ts

@@ -16,12 +16,11 @@
  */
 
 import * as fs from "fs";
-import * as parse5 from "parse5";
+import RewritingStream from "parse5-html-rewriting-stream";
 import * as dom5 from "dom5";
 import {HtmlFileUtils, RedirectsResolver} from "./utils";
 import {Node} from 'dom5';
 import {readMultilineParamFile} from "../utils/command-line";
-import {FileUtils} from "../utils/file-utils";
 import { fail } from "../utils/common";
 import {JSONRedirects} from "./redirects";
 
@@ -34,9 +33,7 @@ import {JSONRedirects} from "./redirects";
  * Additionaly, update some test links (related to web-component-tester)
  */
 
-function main() {
+async function main() {
-  console.log(process.cwd());
-
   if (process.argv.length < 4) {
     console.info("Usage:\n\tnode links_updater.js input_output_param_files redirectFile.json\n");
     process.exit(1);
@@ -50,7 +47,7 @@ function main() {
   for(let i = 0; i < input.length; i += 2) {
     const srcFile = input[i];
     const targetFile = input[i + 1];
-    updater.updateFile(srcFile, targetFile);
+    await updater.updateFile(srcFile, targetFile);
   }
 }
 
@@ -66,29 +63,42 @@ class HtmlFileUpdater {
     isHtmlImport: (node: Node) => node.tagName === "link" && dom5.getAttribute(node, "rel") === "import" &&
         dom5.hasAttribute(node, "href")
   };
+
   public constructor(private readonly redirectsResolver: RedirectsResolver) {
   }
 
-  public updateFile(srcFile: string, targetFile: string) {
+  public async updateFile(srcFile: string, targetFile: string) {
     const html = fs.readFileSync(srcFile, "utf-8");
-    const ast = parse5.parseFragment(html, {locationInfo: true}) as Node;
+    const readStream = fs.createReadStream(srcFile, {encoding: "utf-8"});
-
+    const rewriterOutput = srcFile === targetFile ? targetFile + ".tmp" : targetFile;
-
+    const writeStream = fs.createWriteStream(rewriterOutput, {encoding: "utf-8"});
-    const webComponentTesterImportNode = dom5.query(ast, HtmlFileUpdater.Predicates.isWebComponentTesterImport);
+    const rewriter = new RewritingStream();
-    if(webComponentTesterImportNode) {
+    (rewriter as any).tokenizer.preprocessor.bufferWaterline = Infinity;
-      dom5.setAttribute(webComponentTesterImportNode,  "src", "/components/wct-browser-legacy/browser.js");
+    rewriter.on("startTag", (tag: any) => {
-    }
+      if (HtmlFileUpdater.Predicates.isWebComponentTesterImport(tag)) {
-
+        dom5.setAttribute(tag, "src", "/components/wct-browser-legacy/browser.js");
-    // Update all HTML imports
+      } else if (HtmlFileUpdater.Predicates.isHtmlImport(tag)) {
-    const updateHtmlImportHref = (htmlImportNode: Node) => this.updateRefAttribute(htmlImportNode, srcFile, "href");
+        this.updateRefAttribute(tag, srcFile, "href");
-    dom5.queryAll(ast, HtmlFileUpdater.Predicates.isHtmlImport).forEach(updateHtmlImportHref);
+      } else if (HtmlFileUpdater.Predicates.isScriptWithSrcTag(tag)) {
-
+        this.updateRefAttribute(tag, srcFile, "src");
-    // Update all <script src=...> tags
+      } else {
-    const updateScriptSrc = (scriptTagNode: Node) => this.updateRefAttribute(scriptTagNode, srcFile, "src");
+        const location = tag.sourceCodeLocation;
-    dom5.queryAll(ast, HtmlFileUpdater.Predicates.isScriptWithSrcTag).forEach(updateScriptSrc);
+        const raw = html.substring(location.startOffset, location.endOffset);
-
+        rewriter.emitRaw(raw);
-    const newContent = parse5.serialize(ast);
+        return;
-    FileUtils.writeContent(targetFile, newContent);
+      }
+      rewriter.emitStartTag(tag);
+    });
+    return new Promise<void>((resolve, reject) => {
+      writeStream.on("close", () => {
+        writeStream.close();
+        if (rewriterOutput !== targetFile) {
+          fs.renameSync(rewriterOutput, targetFile);
+        }
+        resolve();
+      });
+      readStream.pipe(rewriter).pipe(writeStream);
+    });
   }
 
   private getResolvedPath(parentHtml: string, href: string) {
@@ -109,14 +119,14 @@ class HtmlFileUpdater {
 
   private updateRefAttribute(node: Node, parentHtml: string, attributeName: string) {
     const ref = dom5.getAttribute(node, attributeName);
-    if(!ref) {
+    if (!ref) {
       fail(`Internal error - ${node} in ${parentHtml} doesn't have attribute ${attributeName}`);
     }
     const newRef = this.getResolvedPath(parentHtml, ref);
-    if(newRef === ref) {
+    if (newRef === ref) {
       return;
     }
-    dom5.setAttribute(node,  attributeName, newRef);
+    dom5.setAttribute(node, attributeName, newRef);
   }
 }
 

tools/node_tools/polygerrit_app_preprocessor/rollup.config.js

@@ -17,5 +17,12 @@
 
 
 export default {
-  external: ['fs', 'path', 'parse5', 'dom5']
+  external: ['fs', 'path', 'parse5', 'dom5', 'parse5-html-rewriting-stream'],
+  onwarn: warn => {
+    // Typescript adds helper methods for await and promise which look like
+    // var __awaiter = (this && this.__awaiter)
+    // It is safe to ignore this warning
+    if(warn.code === 'THIS_IS_UNDEFINED') { return; }
+    throw new Error(warn.message);
+  }
 };

tools/node_tools/yarn.lock

@@ -966,6 +966,26 @@
   dependencies:
     "@types/node" "*"
 
+"@types/parse5-html-rewriting-stream@^5.1.2":
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/@types/parse5-html-rewriting-stream/-/parse5-html-rewriting-stream-5.1.2.tgz#919d5bbf69ef61e11d873e7195891c3811491a03"
+  integrity sha512-7CHY6QlayurvYRST5xatE/ipIueph5V+EW2xU12P0CsNucuwygnuiE4foYsdQUEkhnKrTU62KmikANPnoxiGrg==
+  dependencies:
+    "@types/parse5-sax-parser" "*"
+
+"@types/parse5-sax-parser@*":
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/@types/parse5-sax-parser/-/parse5-sax-parser-5.0.1.tgz#f1e26e82bb09e48cb0c16ff6d1e88aea1e538fd5"
+  integrity sha512-wBEwg10aACLggnb44CwzAA27M1Jrc/8TR16zA61/rKO5XZoi7JSfLjdpXbshsm7wOlM6hpfvwygh40rzM2RsQQ==
+  dependencies:
+    "@types/node" "*"
+    "@types/parse5" "*"
+
+"@types/parse5@*":
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-5.0.2.tgz#a877a4658f8238c8266faef300ae41c84d72ec8a"
+  integrity sha512-BOl+6KDs4ItndUWUFchy3aEqGdHhw0BC4Uu+qoDonN/f0rbUnJbm71Ulj8Tt9jLFRaAxPLKvdS1bBLfx1qXR9g==
+
 "@types/parse5@^0.0.31":
   version "0.0.31"
   resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-0.0.31.tgz#e827a493a443b156e1b582a2e4c3bdc0040f2ee7"
@@ -5803,6 +5823,21 @@ parse-passwd@^1.0.0:
   resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
   integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=
 
+parse5-html-rewriting-stream@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/parse5-html-rewriting-stream/-/parse5-html-rewriting-stream-5.1.1.tgz#fc18570ba0d09b5091250956d1c3f716ef0a07b7"
+  integrity sha512-rbXBeMlJ3pk3tKxLKAUaqvQTZM5KTohXmZvYEv2gU9sQC70w65BxPsh3PVVnwiVNCnNYDtNZRqCKmiMlfdG07Q==
+  dependencies:
+    parse5 "^5.1.1"
+    parse5-sax-parser "^5.1.1"
+
+parse5-sax-parser@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/parse5-sax-parser/-/parse5-sax-parser-5.1.1.tgz#02834a9d08b23ea2d99584841c38be09d5247a15"
+  integrity sha512-9HIh6zd7bF1NJe95LPCUC311CekdOi55R+HWXNCsGY6053DWaMijVKOv1oPvdvPTvFicifZyimBVJ6/qvG039Q==
+  dependencies:
+    parse5 "^5.1.1"
+
 parse5@^1.4.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-1.5.1.tgz#9b7f3b0de32be78dc2401b17573ccaf0f6f59d94"
@@ -5813,6 +5848,11 @@ parse5@^4.0.0:
   resolved "https://registry.yarnpkg.com/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608"
   integrity sha512-VrZ7eOd3T1Fk4XWNXMgiGBK/z0MG48BWG2uQNU4I72fkQuKUTZpl+u9k+CxEG0twMVzSmXEEz12z5Fnw1jIQFA==
 
+parse5@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178"
+  integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==
+
 parseqs@0.0.5:
   version "0.0.5"
   resolved "https://registry.yarnpkg.com/parseqs/-/parseqs-0.0.5.tgz#d5208a3738e46766e291ba2ea173684921a8b89d"