Skip to content

Commit 9318e23

Browse files
petebacondarwinAndrewKushnir
petebacondarwin
authored and
AndrewKushnir
committed
perf(ngcc): use EntryPointManifest to speed up noop ProgramBaseEntryPointFinder (angular#37665)
Previously the `ProgramBasedEntryPointFinder` was parsing all the entry-points referenced by the program for dependencies even if all the entry-points had been processed already. Now this entry-point finder will re-use the `EntryPointManifest` to load the entry-point dependencies when possible which avoids having to parse them all again, on every invocation of ngcc. Previously the `EntryPointManifest` was only used in the `DirectoryWalkerEntryPointFinder`, which also contained the logic for computing the contents of the manifest. This logic has been factored out into an `EntryPointCollector` class. Both the `ProgramBasedEntryPointFinder` and `DirectoryWalkerEntryPointFinder` now use the `EntryPointManifest` and the `EntryPointCollector`. The result of this change is that there is a small cost on the first run of ngcc to compute and store the manifest - the processing takes 102% of the processing time before this PR. But on subsequent runs there is a significant benefit on subsequent runs - the processing takes around 50% of the processing time before this PR. PR Close angular#37665
1 parent 290bc73 commit 9318e23

9 files changed

+497
-331
lines changed

‎packages/compiler-cli/ngcc/src/entry_point_finder/directory_walker_entry_point_finder.ts

+9-138
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,14 @@
55
* Use of this source code is governed by an MIT-style license that can be
66
* found in the LICENSE file at https://angular.io/license
77
*/
8-
import {AbsoluteFsPath, FileSystem, PathSegment} from '../../../src/ngtsc/file_system';
8+
import {AbsoluteFsPath} from '../../../src/ngtsc/file_system';
99
import {Logger} from '../../../src/ngtsc/logging';
1010
import {EntryPointWithDependencies} from '../dependencies/dependency_host';
1111
import {DependencyResolver, SortedEntryPointsInfo} from '../dependencies/dependency_resolver';
12-
import {NgccConfiguration} from '../packages/configuration';
13-
import {getEntryPointInfo, IGNORED_ENTRY_POINT, INCOMPATIBLE_ENTRY_POINT, isEntryPoint, NO_ENTRY_POINT} from '../packages/entry_point';
1412
import {EntryPointManifest} from '../packages/entry_point_manifest';
1513
import {PathMappings} from '../path_mappings';
16-
import {NGCC_DIRECTORY} from '../writing/new_entry_point_file_writer';
1714

15+
import {EntryPointCollector} from './entry_point_collector';
1816
import {EntryPointFinder} from './interface';
1917
import {getBasePaths, trackDuration} from './utils';
2018

@@ -25,9 +23,11 @@ import {getBasePaths, trackDuration} from './utils';
2523
export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
2624
private basePaths = getBasePaths(this.logger, this.sourceDirectory, this.pathMappings);
2725
constructor(
28-
private fs: FileSystem, private config: NgccConfiguration, private logger: Logger,
29-
private resolver: DependencyResolver, private entryPointManifest: EntryPointManifest,
30-
private sourceDirectory: AbsoluteFsPath, private pathMappings: PathMappings|undefined) {}
26+
private logger: Logger, private resolver: DependencyResolver,
27+
private entryPointCollector: EntryPointCollector,
28+
private entryPointManifest: EntryPointManifest, private sourceDirectory: AbsoluteFsPath,
29+
private pathMappings: PathMappings|undefined) {}
30+
3131
/**
3232
* Search the `sourceDirectory`, and sub-directories, using `pathMappings` as necessary, to find
3333
* all package entry-points.
@@ -45,145 +45,16 @@ export class DirectoryWalkerEntryPointFinder implements EntryPointFinder {
4545
/**
4646
* Search the `basePath` for possible Angular packages and entry-points.
4747
*
48-
* @param basePath The path at which to start the search
48+
* @param basePath The path at which to start the search.
4949
* @returns an array of `EntryPoint`s that were found within `basePath`.
5050
*/
5151
walkBasePathForPackages(basePath: AbsoluteFsPath): EntryPointWithDependencies[] {
5252
this.logger.debug(
5353
`No manifest found for ${basePath} so walking the directories for entry-points.`);
5454
const entryPoints = trackDuration(
55-
() => this.walkDirectoryForPackages(basePath),
55+
() => this.entryPointCollector.walkDirectoryForPackages(basePath),
5656
duration => this.logger.debug(`Walking ${basePath} for entry-points took ${duration}s.`));
5757
this.entryPointManifest.writeEntryPointManifest(basePath, entryPoints);
5858
return entryPoints;
5959
}
60-
61-
/**
62-
* Look for Angular packages that need to be compiled, starting at the source directory.
63-
* The function will recurse into directories that start with `@...`, e.g. `@angular/...`.
64-
*
65-
* @param sourceDirectory An absolute path to the root directory where searching begins.
66-
* @returns an array of `EntryPoint`s that were found within `sourceDirectory`.
67-
*/
68-
walkDirectoryForPackages(sourceDirectory: AbsoluteFsPath): EntryPointWithDependencies[] {
69-
// Try to get a primary entry point from this directory
70-
const primaryEntryPoint =
71-
getEntryPointInfo(this.fs, this.config, this.logger, sourceDirectory, sourceDirectory);
72-
73-
// If there is an entry-point but it is not compatible with ngcc (it has a bad package.json or
74-
// invalid typings) then exit. It is unlikely that such an entry point has a dependency on an
75-
// Angular library.
76-
if (primaryEntryPoint === INCOMPATIBLE_ENTRY_POINT) {
77-
return [];
78-
}
79-
80-
const entryPoints: EntryPointWithDependencies[] = [];
81-
if (primaryEntryPoint !== NO_ENTRY_POINT) {
82-
if (primaryEntryPoint !== IGNORED_ENTRY_POINT) {
83-
entryPoints.push(this.resolver.getEntryPointWithDependencies(primaryEntryPoint));
84-
}
85-
this.collectSecondaryEntryPoints(
86-
entryPoints, sourceDirectory, sourceDirectory, this.fs.readdir(sourceDirectory));
87-
88-
// Also check for any nested node_modules in this package but only if at least one of the
89-
// entry-points was compiled by Angular.
90-
if (entryPoints.some(e => e.entryPoint.compiledByAngular)) {
91-
const nestedNodeModulesPath = this.fs.join(sourceDirectory, 'node_modules');
92-
if (this.fs.exists(nestedNodeModulesPath)) {
93-
entryPoints.push(...this.walkDirectoryForPackages(nestedNodeModulesPath));
94-
}
95-
}
96-
97-
return entryPoints;
98-
}
99-
100-
// The `sourceDirectory` was not a package (i.e. there was no package.json)
101-
// So search its sub-directories for Angular packages and entry-points
102-
for (const path of this.fs.readdir(sourceDirectory)) {
103-
if (isIgnorablePath(path)) {
104-
// Ignore hidden files, node_modules and ngcc directory
105-
continue;
106-
}
107-
108-
const absolutePath = this.fs.resolve(sourceDirectory, path);
109-
const stat = this.fs.lstat(absolutePath);
110-
if (stat.isSymbolicLink() || !stat.isDirectory()) {
111-
// Ignore symbolic links and non-directories
112-
continue;
113-
}
114-
115-
entryPoints.push(...this.walkDirectoryForPackages(this.fs.join(sourceDirectory, path)));
116-
}
117-
118-
return entryPoints;
119-
}
120-
121-
/**
122-
* Search the `directory` looking for any secondary entry-points for a package, adding any that
123-
* are found to the `entryPoints` array.
124-
*
125-
* @param entryPoints An array where we will add any entry-points found in this directory
126-
* @param packagePath The absolute path to the package that may contain entry-points
127-
* @param directory The current directory being searched
128-
* @param paths The paths contained in the current `directory`.
129-
*/
130-
private collectSecondaryEntryPoints(
131-
entryPoints: EntryPointWithDependencies[], packagePath: AbsoluteFsPath,
132-
directory: AbsoluteFsPath, paths: PathSegment[]): void {
133-
for (const path of paths) {
134-
if (isIgnorablePath(path)) {
135-
// Ignore hidden files, node_modules and ngcc directory
136-
continue;
137-
}
138-
139-
const absolutePath = this.fs.resolve(directory, path);
140-
const stat = this.fs.lstat(absolutePath);
141-
if (stat.isSymbolicLink()) {
142-
// Ignore symbolic links
143-
continue;
144-
}
145-
146-
const isDirectory = stat.isDirectory();
147-
if (!path.endsWith('.js') && !isDirectory) {
148-
// Ignore files that do not end in `.js`
149-
continue;
150-
}
151-
152-
// If the path is a JS file then strip its extension and see if we can match an
153-
// entry-point (even if it is an ignored one).
154-
const possibleEntryPointPath = isDirectory ? absolutePath : stripJsExtension(absolutePath);
155-
const subEntryPoint =
156-
getEntryPointInfo(this.fs, this.config, this.logger, packagePath, possibleEntryPointPath);
157-
if (isEntryPoint(subEntryPoint)) {
158-
entryPoints.push(this.resolver.getEntryPointWithDependencies(subEntryPoint));
159-
}
160-
161-
if (!isDirectory) {
162-
// This path is not a directory so we are done.
163-
continue;
164-
}
165-
166-
// If not an entry-point itself, this directory may contain entry-points of its own.
167-
const canContainEntryPoints =
168-
subEntryPoint === NO_ENTRY_POINT || subEntryPoint === INCOMPATIBLE_ENTRY_POINT;
169-
const childPaths = this.fs.readdir(absolutePath);
170-
if (canContainEntryPoints &&
171-
childPaths.some(
172-
childPath => childPath.endsWith('.js') &&
173-
this.fs.stat(this.fs.resolve(absolutePath, childPath)).isFile())) {
174-
// We do not consider non-entry-point directories that contain JS files as they are very
175-
// unlikely to be containers for sub-entry-points.
176-
continue;
177-
}
178-
this.collectSecondaryEntryPoints(entryPoints, packagePath, absolutePath, childPaths);
179-
}
180-
}
181-
}
182-
183-
function stripJsExtension<T extends string>(filePath: T): T {
184-
return filePath.replace(/\.js$/, '') as T;
185-
}
186-
187-
function isIgnorablePath(path: PathSegment): boolean {
188-
return path.startsWith('.') || path === 'node_modules' || path === NGCC_DIRECTORY;
18960
}

‎packages/compiler-cli/ngcc/src/entry_point_finder/entry_point_collector.ts

+154
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/**
2+
* @license
3+
* Copyright Google LLC All Rights Reserved.
4+
*
5+
* Use of this source code is governed by an MIT-style license that can be
6+
* found in the LICENSE file at https://angular.io/license
7+
*/
8+
import {AbsoluteFsPath, FileSystem, PathSegment} from '../../../src/ngtsc/file_system';
9+
import {Logger} from '../../../src/ngtsc/logging';
10+
11+
import {EntryPointWithDependencies} from '../dependencies/dependency_host';
12+
import {DependencyResolver} from '../dependencies/dependency_resolver';
13+
import {NgccConfiguration} from '../packages/configuration';
14+
import {getEntryPointInfo, IGNORED_ENTRY_POINT, INCOMPATIBLE_ENTRY_POINT, isEntryPoint, NO_ENTRY_POINT} from '../packages/entry_point';
15+
import {NGCC_DIRECTORY} from '../writing/new_entry_point_file_writer';
16+
17+
/**
18+
* A class that traverses a file-tree, starting at a given path, looking for all entry-points,
19+
* also capturing the dependencies of each entry-point that is found.
20+
*/
21+
export class EntryPointCollector {
22+
constructor(
23+
private fs: FileSystem, private config: NgccConfiguration, private logger: Logger,
24+
private resolver: DependencyResolver) {}
25+
26+
/**
27+
* Look for Angular packages that need to be compiled, starting at the source directory.
28+
* The function will recurse into directories that start with `@...`, e.g. `@angular/...`.
29+
*
30+
* @param sourceDirectory An absolute path to the root directory where searching begins.
31+
* @returns an array of `EntryPoint`s that were found within `sourceDirectory`.
32+
*/
33+
walkDirectoryForPackages(sourceDirectory: AbsoluteFsPath): EntryPointWithDependencies[] {
34+
// Try to get a primary entry point from this directory
35+
const primaryEntryPoint =
36+
getEntryPointInfo(this.fs, this.config, this.logger, sourceDirectory, sourceDirectory);
37+
38+
// If there is an entry-point but it is not compatible with ngcc (it has a bad package.json or
39+
// invalid typings) then exit. It is unlikely that such an entry point has a dependency on an
40+
// Angular library.
41+
if (primaryEntryPoint === INCOMPATIBLE_ENTRY_POINT) {
42+
return [];
43+
}
44+
45+
const entryPoints: EntryPointWithDependencies[] = [];
46+
if (primaryEntryPoint !== NO_ENTRY_POINT) {
47+
if (primaryEntryPoint !== IGNORED_ENTRY_POINT) {
48+
entryPoints.push(this.resolver.getEntryPointWithDependencies(primaryEntryPoint));
49+
}
50+
this.collectSecondaryEntryPoints(
51+
entryPoints, sourceDirectory, sourceDirectory, this.fs.readdir(sourceDirectory));
52+
53+
// Also check for any nested node_modules in this package but only if at least one of the
54+
// entry-points was compiled by Angular.
55+
if (entryPoints.some(e => e.entryPoint.compiledByAngular)) {
56+
const nestedNodeModulesPath = this.fs.join(sourceDirectory, 'node_modules');
57+
if (this.fs.exists(nestedNodeModulesPath)) {
58+
entryPoints.push(...this.walkDirectoryForPackages(nestedNodeModulesPath));
59+
}
60+
}
61+
62+
return entryPoints;
63+
}
64+
65+
// The `sourceDirectory` was not a package (i.e. there was no package.json)
66+
// So search its sub-directories for Angular packages and entry-points
67+
for (const path of this.fs.readdir(sourceDirectory)) {
68+
if (isIgnorablePath(path)) {
69+
// Ignore hidden files, node_modules and ngcc directory
70+
continue;
71+
}
72+
73+
const absolutePath = this.fs.resolve(sourceDirectory, path);
74+
const stat = this.fs.lstat(absolutePath);
75+
if (stat.isSymbolicLink() || !stat.isDirectory()) {
76+
// Ignore symbolic links and non-directories
77+
continue;
78+
}
79+
80+
entryPoints.push(...this.walkDirectoryForPackages(this.fs.join(sourceDirectory, path)));
81+
}
82+
83+
return entryPoints;
84+
}
85+
86+
/**
87+
* Search the `directory` looking for any secondary entry-points for a package, adding any that
88+
* are found to the `entryPoints` array.
89+
*
90+
* @param entryPoints An array where we will add any entry-points found in this directory.
91+
* @param packagePath The absolute path to the package that may contain entry-points.
92+
* @param directory The current directory being searched.
93+
* @param paths The paths contained in the current `directory`.
94+
*/
95+
private collectSecondaryEntryPoints(
96+
entryPoints: EntryPointWithDependencies[], packagePath: AbsoluteFsPath,
97+
directory: AbsoluteFsPath, paths: PathSegment[]): void {
98+
for (const path of paths) {
99+
if (isIgnorablePath(path)) {
100+
// Ignore hidden files, node_modules and ngcc directory
101+
continue;
102+
}
103+
104+
const absolutePath = this.fs.resolve(directory, path);
105+
const stat = this.fs.lstat(absolutePath);
106+
if (stat.isSymbolicLink()) {
107+
// Ignore symbolic links
108+
continue;
109+
}
110+
111+
const isDirectory = stat.isDirectory();
112+
if (!path.endsWith('.js') && !isDirectory) {
113+
// Ignore files that do not end in `.js`
114+
continue;
115+
}
116+
117+
// If the path is a JS file then strip its extension and see if we can match an
118+
// entry-point (even if it is an ignored one).
119+
const possibleEntryPointPath = isDirectory ? absolutePath : stripJsExtension(absolutePath);
120+
const subEntryPoint =
121+
getEntryPointInfo(this.fs, this.config, this.logger, packagePath, possibleEntryPointPath);
122+
if (isEntryPoint(subEntryPoint)) {
123+
entryPoints.push(this.resolver.getEntryPointWithDependencies(subEntryPoint));
124+
}
125+
126+
if (!isDirectory) {
127+
// This path is not a directory so we are done.
128+
continue;
129+
}
130+
131+
// If not an entry-point itself, this directory may contain entry-points of its own.
132+
const canContainEntryPoints =
133+
subEntryPoint === NO_ENTRY_POINT || subEntryPoint === INCOMPATIBLE_ENTRY_POINT;
134+
const childPaths = this.fs.readdir(absolutePath);
135+
if (canContainEntryPoints &&
136+
childPaths.some(
137+
childPath => childPath.endsWith('.js') &&
138+
this.fs.stat(this.fs.resolve(absolutePath, childPath)).isFile())) {
139+
// We do not consider non-entry-point directories that contain JS files as they are very
140+
// unlikely to be containers for sub-entry-points.
141+
continue;
142+
}
143+
this.collectSecondaryEntryPoints(entryPoints, packagePath, absolutePath, childPaths);
144+
}
145+
}
146+
}
147+
148+
function stripJsExtension<T extends string>(filePath: T): T {
149+
return filePath.replace(/\.js$/, '') as T;
150+
}
151+
152+
function isIgnorablePath(path: PathSegment): boolean {
153+
return path.startsWith('.') || path === 'node_modules' || path === NGCC_DIRECTORY;
154+
}

0 commit comments

Comments
 (0)