feat(build): Move HTML copying into the broccoli task.

This includes all tasks to construct a Dart tree, except for formatting, and
reverse engineers/refactors the various copy tools for added more sanity.

Author: mprobst

gulpfile.js

@@ -294,7 +294,7 @@ gulp.task('build/clean.docs', clean(gulp, gulpPlugins, {
 // ------------
 // transpile
 
-gulp.task('build/transpile.dart', ['build.broccoli.tools'], function() {
+gulp.task('build/tree.dart', ['build.broccoli.tools'], function() {
   return getBroccoli().forDartTree().buildOnce();
 });
 
@@ -617,12 +617,7 @@ gulp.task('test.transpiler.unittest', function() {
 
 // Builds all Dart packages, but does not compile them
 gulp.task('build/packages.dart', function(done) {
-  runSequence(
-    'build/transpile.dart', // Creates the folder structure needed by subsequent tasks.
-    ['build/html.dart', 'build/copy.dart', 'build/multicopy.dart'],
-    'build/format.dart',
-    done
-  );
+  runSequence('build/tree.dart', 'build/format.dart', done);
 });
 
 // Builds and compiles all Dart packages

tools/broccoli/broccoli-filter.d.ts

@@ -1,8 +1,11 @@
 /// <reference path="../typings/es6-promise/es6-promise.d.ts" />
 
+interface FilterOptions {
+  extensions?: string[]
+}
 
 declare class Filter {
-  constructor(inputTree: any);
+  constructor(inputTree: any, options?: FilterOptions);
   processString(contents: string, relativePath: string): string;
   // NB: This function is probably not intended as part of the public API
   processFile(srcDir: string, destDir: string, relativePath: string): Promise<any>;

tools/broccoli/trees/dart_tree.ts

@@ -1,32 +1,65 @@
+/// <reference path="../../typings/node/node.d.ts" />
 'use strict';
 
+import {MultiCopy} from './multi_copy';
 var Funnel = require('broccoli-funnel');
+var glob = require('glob');
+var mergeTrees = require('broccoli-merge-trees');
 var path = require('path');
+var renderLodashTemplate = require('broccoli-lodash');
 var replace = require('broccoli-replace');
 var stew = require('broccoli-stew');
 var ts2dart = require('../broccoli-ts2dart');
 
-module.exports = function makeDartTree() {
-  // Transpile everything in 'modules'...
-  var modulesTree = new Funnel('modules', {
-    include: ['**/*.js', '**/*.ts', '**/*.dart'],  // .dart file available means don't translate.
-    exclude: ['rtts_assert/**/*'],  // ... except for the rtts_asserts (don't apply to Dart).
-    destDir: '/'                    // Remove the 'modules' prefix.
-  });
+/**
+ * A funnel starting at modules, including the given filters, and moving into the root.
+ * @param include Include glob filters.
+ */
+function modulesFunnel(include: string[], exclude?: string[]) {
+  return new Funnel('modules', {include, destDir: '/', exclude});
+}
+
+/**
+ * Replaces $SCRIPT$ in .html files with actual <script> tags.
+ */
+function replaceScriptTagInHtml(content: string, relativePath: string): string {
+  var scriptTags = '';
+  if (relativePath.match(/^benchmarks/)) {
+    scriptTags += '<script src="url_params_to_form.js" type="text/javascript"></script>\n';
+  }
+  var scriptName = relativePath.replace(/.*\/([^/]+)\.html$/, '$1.dart');
+  scriptTags += '<script src="' + scriptName + '" type="application/dart"></script>\n' +
+                '<script src="packages/browser/dart.js" type="text/javascript"></script>';
+  return content.replace('$SCRIPTS$', scriptTags);
+}
+
+function stripModulePrefix(relativePath: string): string {
+  if (!relativePath.match(/^modules\//)) {
+    throw new Error('Expected path to root at modules/: ' + relativePath);
+  }
+  return relativePath.replace(/^modules\//, '');
+}
 
-  // Transpile to dart.
-  var dartTree = ts2dart.transpile(modulesTree);
+function getSourceTree() {
+  // Transpile everything in 'modules' except for rtts_assertions.
+  var tsInputTree = modulesFunnel(['**/*.js', '**/*.ts', '**/*.dart'], ['rtts_assert/**/*']);
+  var transpiled = ts2dart.transpile(tsInputTree);
+  // Native sources, dart only examples, etc.
+  var dartSrcs = modulesFunnel(['**/*.dart']);
+  return mergeTrees([transpiled, dartSrcs]);
+}
 
+function fixDartFolderLayout(sourceTree) {
   // Move around files to match Dart's layout expectations.
-  dartTree = stew.rename(dartTree, function(relativePath) {
+  return stew.rename(sourceTree, function(relativePath) {
     // If a file matches the `pattern`, insert the given `insertion` as the second path part.
     var replacements = [
       {pattern: /^benchmarks\/test\//, insertion: ''},
       {pattern: /^benchmarks\//, insertion: 'web'},
       {pattern: /^benchmarks_external\/test\//, insertion: ''},
       {pattern: /^benchmarks_external\//, insertion: 'web'},
-      {pattern: /^example.?\//, insertion: 'web/'},
-      {pattern: /^example.?\/test\//, insertion: ''},
+      {pattern: /^examples\/test\//, insertion: ''},
+      {pattern: /^examples\//, insertion: 'web/'},
       {pattern: /^[^\/]*\/test\//, insertion: ''},
       {pattern: /^./, insertion: 'lib'},  // catch all.
     ];
@@ -41,7 +74,75 @@ module.exports = function makeDartTree() {
     }
     throw new Error('Failed to match any path: ' + relativePath);
   });
+}
+
+function getHtmlSourcesTree() {
+  // Replace $SCRIPT$ markers in HTML files.
+  var htmlSrcsTree = stew.map(modulesFunnel(['*/src/**/*.html']), replaceScriptTagInHtml);
+  // Copy a url_params_to_form.js for each benchmark html file.
+  var urlParamsToFormTree = new MultiCopy('', {
+    srcPath: 'tools/build/snippets/url_params_to_form.js',
+    targetPatterns: ['modules/benchmarks*/src/*', 'modules/benchmarks*/src/*/*'],
+  });
+  urlParamsToFormTree = stew.rename(urlParamsToFormTree, stripModulePrefix);
+  return mergeTrees([htmlSrcsTree, urlParamsToFormTree]);
+}
+
+
+function getTemplatedPubspecsTree() {
+  // The JSON structure for templating pubspec.yaml files.
+  var BASE_PACKAGE_JSON = require('../../../package.json');
+  var COMMON_PACKAGE_JSON = {
+    version: BASE_PACKAGE_JSON.version,
+    homepage: BASE_PACKAGE_JSON.homepage,
+    bugs: BASE_PACKAGE_JSON.bugs,
+    license: BASE_PACKAGE_JSON.license,
+    contributors: BASE_PACKAGE_JSON.contributors,
+    dependencies: BASE_PACKAGE_JSON.dependencies,
+    devDependencies: {
+      "yargs": BASE_PACKAGE_JSON.devDependencies['yargs'],
+      "gulp-sourcemaps": BASE_PACKAGE_JSON.devDependencies['gulp-sourcemaps'],
+      "gulp-traceur": BASE_PACKAGE_JSON.devDependencies['gulp-traceur'],
+      "gulp": BASE_PACKAGE_JSON.devDependencies['gulp'],
+      "gulp-rename": BASE_PACKAGE_JSON.devDependencies['gulp-rename'],
+      "through2": BASE_PACKAGE_JSON.devDependencies['through2']
+    }
+  };
+  // Generate pubspec.yaml from templates.
+  // Lodash insists on dropping one level of extension, so first artificially rename the yaml
+  // files to .yaml.template.
+  var pubspecs = stew.rename(modulesFunnel(['**/pubspec.yaml']), '.yaml', '.yaml.template');
+  // Then render the templates.
+  return renderLodashTemplate(
+      pubspecs,
+      {files: ['**/pubspec.yaml.template'], context: {'packageJson': COMMON_PACKAGE_JSON}});
+}
+
+function getDocsTree() {
+  // LICENSE files
+  var licenses = new MultiCopy('', {
+    srcPath: 'LICENSE',
+    targetPatterns: ['modules/*'],
+    exclude: ['*/rtts_assert'],  // Not in dart.
+  });
+  licenses = stew.rename(licenses, stripModulePrefix);
+
+  // Documentation.
+  // Rename *.dart.md -> *.dart.
+  var mdTree = stew.rename(modulesFunnel(['**/*.dart.md']),
+                           relativePath => relativePath.replace(/\.dart\.md$/, '.md'));
+  // Copy all assets, ignore .js. and .dart. (handled above).
+  var docs = modulesFunnel(['**/*.md', '**/*.png', '**/*.html', '**/*.css'],
+                           ['**/*.js.md', '**/*.dart.md']);
+  return mergeTrees([licenses, mdTree, docs]);
+}
+
+module.exports = function makeDartTree() {
+  var sourceTree = mergeTrees([getSourceTree(), getHtmlSourcesTree()]);
+  sourceTree = fixDartFolderLayout(sourceTree);
+
+  var mergedResult = mergeTrees([sourceTree, getTemplatedPubspecsTree(), getDocsTree()]);
 
   // Move the tree under the 'dart' folder.
-  return stew.mv(dartTree, 'dart');
+  return stew.mv(mergedResult, 'dart');
 };

tools/broccoli/trees/multi_copy.ts

@@ -0,0 +1,47 @@
+/// <reference path="../../typings/node/node.d.ts" />
+/// <reference path="../../typings/fs-extra/fs-extra.d.ts" />
+import Writer = require('broccoli-writer');
+import fs = require('fs');
+import fsx = require('fs-extra');
+var minimatch = require('minimatch');
+var path = require('path');
+var glob = require('glob');
+
+export interface MultiCopyOptions {
+  /** The path of the file to copy. */
+  srcPath: string,
+  /** A list of glob patterns of folders to copy to, matched against the input tree. */
+  targetPatterns: string[],
+  /** List of glob patterns to *not* copy to, matched against the matches from `targetPatterns`. */
+  exclude?: string[],
+}
+
+/**
+ * A writer that copies an input file from an input path into (potentially many) output locations
+ * given by glob patterns, .
+ */
+export class MultiCopy extends Writer {
+  constructor(private inputTree, private options: MultiCopyOptions) { super(); }
+
+  write(readTree: (tree) => Promise<string>, destDir: string): Promise<any> {
+    return readTree(this.inputTree)
+        .then((inputPath: string) => {
+          var fileName = path.basename(this.options.srcPath);
+          var data = fs.readFileSync(path.join(inputPath, this.options.srcPath), 'utf-8');
+
+          this.options.targetPatterns.forEach(pattern => {
+            var paths: string[] = glob.sync(pattern);
+            paths = paths.filter(p => fs.statSync(p).isDirectory());
+            if (this.options.exclude) {
+              paths = paths.filter(p => !this.options.exclude.some((excl) => minimatch(p, excl)));
+            }
+            paths.forEach(p => {
+              var folder = path.join(destDir, p);
+              fsx.mkdirsSync(folder);
+              var outputPath = path.join(folder, fileName);
+              fs.writeFileSync(outputPath, data);
+            });
+          });
+        });
+  }
+}