perf(compiler): speed up watch mode (angular#19275)

- don’t regenerate code for .d.ts files when
  an oldProgram is passed to `createProgram`
- cache `fileExists` / `getSourceFile` / `readFile` in watch mode
- refactor tests to share common code in `test_support`
- support `—diagnostic` command line to print total time
  used per watch mode compilation.
PR Close angular#19275

Author: tbosch

packages/compiler-cli/src/compiler_host.ts

@@ -260,7 +260,7 @@ export class CompilerHost extends BaseAotCompilerHost<CompilerHostContext> {
     this.urlResolver = createOfflineCompileUrlResolver();
   }
 
-  getMetadataForSourceFile(filePath: string): ModuleMetadata|undefined {
+  protected getSourceFile(filePath: string): ts.SourceFile {
     let sf = this.program.getSourceFile(filePath);
     if (!sf) {
       if (this.context.fileExists(filePath)) {
@@ -270,7 +270,11 @@ export class CompilerHost extends BaseAotCompilerHost<CompilerHostContext> {
         throw new Error(`Source file ${filePath} not present in program.`);
       }
     }
-    return this.metadataProvider.getMetadata(sf);
+    return sf;
+  }
+
+  getMetadataForSourceFile(filePath: string): ModuleMetadata|undefined {
+    return this.metadataProvider.getMetadata(this.getSourceFile(filePath));
   }
 
   toSummaryFileName(fileName: string, referringSrcFileName: string): string {

packages/compiler-cli/src/ngtools_api2.ts

@@ -66,8 +66,9 @@ export interface CompilerHost extends ts.CompilerHost {
 export enum EmitFlags {
   DTS = 1 << 0,
   JS = 1 << 1,
+  Codegen = 1 << 4,
 
-  Default = DTS | JS
+  Default = DTS | JS | Codegen
 }
 
 export interface CustomTransformers {
@@ -106,6 +107,7 @@ export interface Program {
     customTransformers?: CustomTransformers,
     emitCallback?: TsEmitCallback
   }): ts.EmitResult;
+  getLibrarySummaries(): {fileName: string, content: string}[];
 }
 
 // Wrapper for createProgram.

packages/compiler-cli/src/perform_watch.ts

@@ -35,18 +35,35 @@ const ChangeDiagnostics = {
   },
 };
 
+function totalCompilationTimeDiagnostic(timeInMillis: number): api.Diagnostic {
+  let duration: string;
+  if (timeInMillis > 1000) {
+    duration = `${(timeInMillis / 1000).toPrecision(2)}s`;
+  } else {
+    duration = `${timeInMillis}ms`;
+  }
+  return {
+    category: ts.DiagnosticCategory.Message,
+    messageText: `Total time: ${duration}`,
+    code: api.DEFAULT_ERROR_CODE,
+    source: api.SOURCE,
+  };
+}
+
 export enum FileChangeEvent {
   Change,
-  CreateDelete
+  CreateDelete,
+  CreateDeleteDir,
 }
 
 export interface PerformWatchHost {
   reportDiagnostics(diagnostics: Diagnostics): void;
   readConfiguration(): ParsedConfiguration;
   createCompilerHost(options: api.CompilerOptions): api.CompilerHost;
   createEmitCallback(options: api.CompilerOptions): api.TsEmitCallback|undefined;
-  onFileChange(listener: (event: FileChangeEvent, fileName: string) => void):
-      {close: () => void, ready: (cb: () => void) => void};
+  onFileChange(
+      options: api.CompilerOptions, listener: (event: FileChangeEvent, fileName: string) => void,
+      ready: () => void): {close: () => void};
   setTimeout(callback: () => void, ms: number): any;
   clearTimeout(timeoutId: any): void;
 }
@@ -60,23 +77,17 @@ export function createPerformWatchHost(
     createCompilerHost: options => createCompilerHost({options}),
     readConfiguration: () => readConfiguration(configFileName, existingOptions),
     createEmitCallback: options => createEmitCallback ? createEmitCallback(options) : undefined,
-    onFileChange: (listeners) => {
-      const parsed = readConfiguration(configFileName, existingOptions);
-      function stubReady(cb: () => void) { process.nextTick(cb); }
-      if (parsed.errors && parsed.errors.length) {
-        reportDiagnostics(parsed.errors);
-        return {close: () => {}, ready: stubReady};
-      }
-      if (!parsed.options.basePath) {
+    onFileChange: (options, listener, ready: () => void) => {
+      if (!options.basePath) {
         reportDiagnostics([{
           category: ts.DiagnosticCategory.Error,
           messageText: 'Invalid configuration option. baseDir not specified',
           source: api.SOURCE,
           code: api.DEFAULT_ERROR_CODE
         }]);
-        return {close: () => {}, ready: stubReady};
+        return {close: () => {}};
       }
-      const watcher = chokidar.watch(parsed.options.basePath, {
+      const watcher = chokidar.watch(options.basePath, {
         // ignore .dotfiles, .js and .map files.
         // can't ignore other files as we e.g. want to recompile if an `.html` file changes as well.
         ignored: /((^[\/\\])\..)|(\.js$)|(\.map$)|(\.metadata\.json)/,
@@ -86,22 +97,32 @@ export function createPerformWatchHost(
       watcher.on('all', (event: string, path: string) => {
         switch (event) {
           case 'change':
-            listeners(FileChangeEvent.Change, path);
+            listener(FileChangeEvent.Change, path);
             break;
           case 'unlink':
           case 'add':
-            listeners(FileChangeEvent.CreateDelete, path);
+            listener(FileChangeEvent.CreateDelete, path);
+            break;
+          case 'unlinkDir':
+          case 'addDir':
+            listener(FileChangeEvent.CreateDeleteDir, path);
             break;
         }
       });
-      function ready(cb: () => void) { watcher.on('ready', cb); }
+      watcher.on('ready', ready);
       return {close: () => watcher.close(), ready};
     },
     setTimeout: (ts.sys.clearTimeout && ts.sys.setTimeout) || setTimeout,
     clearTimeout: (ts.sys.setTimeout && ts.sys.clearTimeout) || clearTimeout,
   };
 }
 
+interface CacheEntry {
+  exists?: boolean;
+  sf?: ts.SourceFile;
+  content?: string;
+}
+
 /**
  * The logic in this function is adapted from `tsc.ts` from TypeScript.
  */
@@ -112,16 +133,30 @@ export function performWatchCompilation(host: PerformWatchHost):
   let cachedOptions: ParsedConfiguration|undefined;  // CompilerOptions cached from last compilation
   let timerHandleForRecompilation: any;  // Handle for 0.25s wait timer to trigger recompilation
 
-  // Watch basePath, ignoring .dotfiles
-  const fileWatcher = host.onFileChange(watchedFileChanged);
   const ingoreFilesForWatch = new Set<string>();
+  const fileCache = new Map<string, CacheEntry>();
 
   const firstCompileResult = doCompilation();
 
-  const readyPromise = new Promise(resolve => fileWatcher.ready(resolve));
+  // Watch basePath, ignoring .dotfiles
+  let resolveReadyPromise: () => void;
+  const readyPromise = new Promise(resolve => resolveReadyPromise = resolve);
+  // Note: ! is ok as options are filled after the first compilation
+  // Note: ! is ok as resolvedReadyPromise is filled by the previous call
+  const fileWatcher =
+      host.onFileChange(cachedOptions !.options, watchedFileChanged, resolveReadyPromise !);
 
   return {close, ready: cb => readyPromise.then(cb), firstCompileResult};
 
+  function cacheEntry(fileName: string): CacheEntry {
+    let entry = fileCache.get(fileName);
+    if (!entry) {
+      entry = {};
+      fileCache.set(fileName, entry);
+    }
+    return entry;
+  }
+
   function close() {
     fileWatcher.close();
     if (timerHandleForRecompilation) {
@@ -139,11 +174,8 @@ export function performWatchCompilation(host: PerformWatchHost):
       host.reportDiagnostics(cachedOptions.errors);
       return cachedOptions.errors;
     }
+    const startTime = Date.now();
     if (!cachedCompilerHost) {
-      // TODO(chuckj): consider avoiding re-generating factories for libraries.
-      // Consider modifying the AotCompilerHost to be able to remember the summary files
-      // generated from previous compiliations and return false from isSourceFile for
-      // .d.ts files for which a summary file was already generated.å
       cachedCompilerHost = host.createCompilerHost(cachedOptions.options);
       const originalWriteFileCallback = cachedCompilerHost.writeFile;
       cachedCompilerHost.writeFile = function(
@@ -152,6 +184,31 @@ export function performWatchCompilation(host: PerformWatchHost):
         ingoreFilesForWatch.add(path.normalize(fileName));
         return originalWriteFileCallback(fileName, data, writeByteOrderMark, onError, sourceFiles);
       };
+      const originalFileExists = cachedCompilerHost.fileExists;
+      cachedCompilerHost.fileExists = function(fileName: string) {
+        const ce = cacheEntry(fileName);
+        if (ce.exists == null) {
+          ce.exists = originalFileExists.call(this, fileName);
+        }
+        return ce.exists !;
+      };
+      const originalGetSourceFile = cachedCompilerHost.getSourceFile;
+      cachedCompilerHost.getSourceFile = function(
+          fileName: string, languageVersion: ts.ScriptTarget) {
+        const ce = cacheEntry(fileName);
+        if (!ce.sf) {
+          ce.sf = originalGetSourceFile.call(this, fileName, languageVersion);
+        }
+        return ce.sf !;
+      };
+      const originalReadFile = cachedCompilerHost.readFile;
+      cachedCompilerHost.readFile = function(fileName: string) {
+        const ce = cacheEntry(fileName);
+        if (ce.content == null) {
+          ce.content = originalReadFile.call(this, fileName);
+        }
+        return ce.content !;
+      };
     }
     ingoreFilesForWatch.clear();
     const compileResult = performCompilation({
@@ -166,6 +223,11 @@ export function performWatchCompilation(host: PerformWatchHost):
       host.reportDiagnostics(compileResult.diagnostics);
     }
 
+    const endTime = Date.now();
+    if (cachedOptions.options.diagnostics) {
+      const totalTime = (endTime - startTime) / 1000;
+      host.reportDiagnostics([totalCompilationTimeDiagnostic(endTime - startTime)]);
+    }
     const exitCode = exitCodeFromResult(compileResult.diagnostics);
     if (exitCode == 0) {
       cachedProgram = compileResult.program;
@@ -191,11 +253,19 @@ export function performWatchCompilation(host: PerformWatchHost):
         path.normalize(fileName) === path.normalize(cachedOptions.project)) {
       // If the configuration file changes, forget everything and start the recompilation timer
       resetOptions();
-    } else if (event === FileChangeEvent.CreateDelete) {
+    } else if (
+        event === FileChangeEvent.CreateDelete || event === FileChangeEvent.CreateDeleteDir) {
       // If a file was added or removed, reread the configuration
       // to determine the new list of root files.
       cachedOptions = undefined;
     }
+
+    if (event === FileChangeEvent.CreateDeleteDir) {
+      fileCache.clear();
+    } else {
+      fileCache.delete(fileName);
+    }
+
     if (!ingoreFilesForWatch.has(path.normalize(fileName))) {
       // Ignore the file if the file is one that was written by the compiler.
       startTimerForRecompilation();

packages/compiler-cli/src/transformers/api.ts

@@ -30,6 +30,11 @@ export function isNgDiagnostic(diagnostic: any): diagnostic is Diagnostic {
 }
 
 export interface CompilerOptions extends ts.CompilerOptions {
+  // Write statistics about compilation (e.g. total time, ...)
+  // Note: this is the --diagnostics command line option from TS (which is @internal
+  // on ts.CompilerOptions interface).
+  diagnostics?: boolean;
+
   // Absolute path to a directory where generated file structure is written.
   // If unspecified, generated files will be written alongside sources.
   // @deprecated - no effect
@@ -273,4 +278,10 @@ export interface Program {
     customTransformers?: CustomTransformers,
     emitCallback?: TsEmitCallback
   }): ts.EmitResult;
+
+  /**
+   * Returns the .ngsummary.json files of libraries that have been compiled
+   * in this program or previous programs.
+   */
+  getLibrarySummaries(): {fileName: string, content: string}[];
 }

packages/compiler-cli/src/transformers/compiler_host.ts

@@ -61,7 +61,8 @@ export class TsCompilerAotCompilerTypeCheckHostAdapter extends
   constructor(
       private rootFiles: string[], options: CompilerOptions, context: CompilerHost,
       private metadataProvider: MetadataProvider,
-      private codeGenerator: (fileName: string) => GeneratedFile[]) {
+      private codeGenerator: (fileName: string) => GeneratedFile[],
+      private summariesFromPreviousCompilations: Map<string, string>) {
     super(options, context);
     this.moduleResolutionCache = ts.createModuleResolutionCache(
         this.context.getCurrentDirectory !(), this.context.getCanonicalFileName.bind(this.context));
@@ -292,7 +293,8 @@ export class TsCompilerAotCompilerTypeCheckHostAdapter extends
     }
     this.generatedCodeFor.add(fileName);
 
-    const baseNameFromGeneratedFile = this._getBaseNameForGeneratedFile(fileName);
+    const baseNameFromGeneratedFile = this._getBaseNamesForGeneratedFile(fileName).find(
+        fileName => this.isSourceFile(fileName) && this.fileExists(fileName));
     if (baseNameFromGeneratedFile) {
       return this.ensureCodeGeneratedFor(baseNameFromGeneratedFile);
     }
@@ -336,29 +338,58 @@ export class TsCompilerAotCompilerTypeCheckHostAdapter extends
     fileName = stripNgResourceSuffix(fileName);
     // Note: Don't rely on this.generatedSourceFiles here,
     // as it might not have been filled yet.
-    if (this._getBaseNameForGeneratedFile(fileName)) {
+    if (this._getBaseNamesForGeneratedFile(fileName).find(baseFileName => {
+          if (this.isSourceFile(baseFileName)) {
+            return this.fileExists(baseFileName);
+          } else {
+            // Note: the factories of a previous program
+            // are not reachable via the regular fileExists
+            // as they might be in the outDir. So we derive their
+            // fileExist information based on the .ngsummary.json file.
+            return this.fileExists(summaryFileName(baseFileName));
+          }
+        })) {
       return true;
     }
-    return this.originalSourceFiles.has(fileName) || this.context.fileExists(fileName);
+    return this.summariesFromPreviousCompilations.has(fileName) ||
+        this.originalSourceFiles.has(fileName) || this.context.fileExists(fileName);
   }
 
-  private _getBaseNameForGeneratedFile(genFileName: string): string|null {
+  private _getBaseNamesForGeneratedFile(genFileName: string): string[] {
     const genMatch = GENERATED_FILES.exec(genFileName);
     if (genMatch) {
       const [, base, genSuffix, suffix] = genMatch;
-      // Note: on-the-fly generated files always have a `.ts` suffix,
-      // but the file from which we generated it can be a `.ts`/ `.d.ts`
-      // (see options.generateCodeForLibraries).
-      // It can also be a `.css` file in case of a `.css.ngstyle.ts` file
-      if (suffix === 'ts') {
-        const baseNames =
-            genSuffix.indexOf('ngstyle') >= 0 ? [base] : [`${base}.ts`, `${base}.d.ts`];
-        return baseNames.find(
-                   baseName => this.isSourceFile(baseName) && this.fileExists(baseName)) ||
-            null;
+      let baseNames: string[] = [];
+      if (genSuffix.indexOf('ngstyle') >= 0) {
+        // Note: ngstlye files have names like `afile.css.ngstyle.ts`
+        baseNames = [base];
+      } else if (suffix === 'd.ts') {
+        baseNames = [base + '.d.ts'];
+      } else if (suffix === 'ts') {
+        // Note: on-the-fly generated files always have a `.ts` suffix,
+        // but the file from which we generated it can be a `.ts`/ `.d.ts`
+        // (see options.generateCodeForLibraries).
+        baseNames = [`${base}.ts`, `${base}.d.ts`];
       }
+      return baseNames;
     }
-    return null;
+    return [];
+  }
+
+  loadSummary(filePath: string): string|null {
+    if (this.summariesFromPreviousCompilations.has(filePath)) {
+      return this.summariesFromPreviousCompilations.get(filePath) !;
+    }
+    return super.loadSummary(filePath);
+  }
+
+  isSourceFile(filePath: string): boolean {
+    // If we have a summary from a previous compilation,
+    // treat the file never as a source file.
+    if (this.summariesFromPreviousCompilations.has(summaryFileName(filePath))) {
+      return false;
+    }
+    return super.isSourceFile(filePath);
   }
 
   readFile = (fileName: string) => this.context.readFile(fileName);
@@ -431,3 +462,7 @@ function stripNgResourceSuffix(fileName: string): string {
 function addNgResourceSuffix(fileName: string): string {
   return `${fileName}.$ngresource$`;
 }
+
+function summaryFileName(fileName: string): string {
+  return fileName.replace(EXT, '') + '.ngsummary.json';
+}