First pass at build-shred-map complete

Author: jtraband

gulpfile.js

@@ -3,13 +3,14 @@ var watch = require('gulp-watch');
 var gutil = require('gulp-util');
 var path = require('path');
 var del = require('del');
+var _ = require('lodash');
 
 var docShredder = require('./public/doc-shredder/doc-shredder');
 
 var _shredOptions =  {
   basePath: path.resolve('./public/docs'),
-  sourceDir: "_examples",
-  destDir:  "_fragments"
+  examplesDir: "_examples",
+  fragmentsDir:  "_fragments"
 };
 
 /*
@@ -42,19 +43,29 @@ gulp.task('serve-and-watch', function (cb) {
 });
 
 gulp.task('shred-full', ['shred-clean'], function() {
-  docShredder.shred( _shredOptions);
+  return docShredder.shred( _shredOptions);
 });
 
 gulp.task('shred-clean', function(cb) {
-  var cleanPath = path.join(_shredOptions.basePath, _shredOptions.destDir, '**/*.*')
+  var cleanPath = path.join(_shredOptions.basePath, _shredOptions.fragmentsDir, '**/*.*')
   del([ cleanPath, '!**/*.ovr.*'], function (err, paths) {
     // console.log('Deleted files/folders:\n', paths.join('\n'));
     cb();
   });
 });
 
+gulp.task('build-shred-maps', ['shred-full'], function() {
+  var options = _.extend(_shredOptions, {
+    jadeDir: '.',
+    outputDir: '.'
+  });
+  return docShredder.buildShredMap(options).then(function(x) {
+    // var json = x[2];
+  });
+})
+
 function shredWatch(shredOptions, postShredAction) {
-  var pattern = path.join(shredOptions.basePath, shredOptions.sourceDir, "**/*.*");
+  var pattern = path.join(shredOptions.basePath, shredOptions.examplesDir, "**/*.*");
   watch([pattern], function (event, done) {
     console.log('Event type: ' + event.event); // added, changed, or deleted
     console.log('Event path: ' + event.path); // The path of the modified file

public/doc-shredder/doc-shredder.js

@@ -7,28 +7,7 @@ var delPromise =  Q.denodeify(del);
 var Dgeni = require('dgeni');
 var _ = require('lodash');
 
-var resolveShredOptions = function(shredOptions) {
-  return _.defaults({}, shredOptions, {
-    basePath: path.resolve('.'),
-    // read files from any subdir under here
-    sourceDir: "docs/_examples",
-    // shredded files get copied here with same subdir structure.
-    destDir: "docs/_fragments",
-    // whether to include subdirectories when shredding.
-    includeSubdirs: true
-  });
-}
 
-var resolveMapOptions = function(mapOptions) {
-  return _.defaults({}, mapOptions, {
-    basePath: path.resolve('.'),
-    // read files from any subdir under here
-    sourceDir: "docs",
-    destDir: "docs",
-    // whether to include subdirectories when shredding.
-    includeSubdirs: true
-  });
-}
 
 var shred = function(shredOptions) {
   try {
@@ -42,26 +21,26 @@ var shred = function(shredOptions) {
 }
 
 var shredSingleDir = function(shredOptions, filePath) {
-  shredOptions = resolveOptions(shredOptions);
-  var root = path.resolve(shredOptions.basePath, shredOptions.sourceDir);
+  shredOptions = resolveShredOptions(shredOptions);
+  var root = path.resolve(shredOptions.basePath, shredOptions.examplesDir);
   var fileDir = path.dirname(filePath);
   var relativePath = path.relative(root, fileDir);
-  var sourceDir = path.join(shredOptions.sourceDir, relativePath);
-  var destDir = path.join(shredOptions.destDir, relativePath);
+  var examplesDir = path.join(shredOptions.examplesDir, relativePath);
+  var fragmentsDir = path.join(shredOptions.fragmentsDir, relativePath);
   var options = {
     basePath: shredOptions.basePath,
     includeSubdirs: false,
-    sourceDir: sourceDir,
-    destDir:  destDir
+    examplesDir: examplesDir,
+    fragmentsDir: fragmentsDir
   }
-  var cleanPath = path.join(shredOptions.basePath, destDir, '*.*')
+  var cleanPath = path.join(shredOptions.basePath, fragmentsDir, '*.*')
   return delPromise([ cleanPath, '!**/*.ovr.*']).then(function(paths) {
     // console.log('Deleted files/folders:\n', paths.join('\n'));
     return shred(options);
   });
 }
 
-var getShredMap = function(shredMapOptions) {
+var buildShredMap = function(shredMapOptions) {
   try {
     var pkg = createShredMapPackage(shredMapOptions);
     var dgeni = new Dgeni([ pkg]);
@@ -73,11 +52,11 @@ var getShredMap = function(shredMapOptions) {
 }
 
 
+
 module.exports = {
   shred: shred,
   shredSingleDir: shredSingleDir,
-  resolveShredOptions: resolveShredOptions,
-  getShredMap: getShredMap
+  buildShredMap: buildShredMap
 };
 
 function createShredPackage(shredOptions) {
@@ -103,28 +82,28 @@ function createShredPackage(shredOptions) {
       var extns = ['*.js', '*.html', '*.ts', '*.css' ];
       var includeFiles = extns.map(function(extn) {
         if (options.includeSubdirs) {
-          return path.join(options.sourceDir, '**', extn);
+          return path.join(options.examplesDir, '**', extn);
         } else {
-          return path.join(options.sourceDir, extn);
+          return path.join(options.examplesDir, extn);
         }
       });
       readFilesProcessor.sourceFiles = [ {
         // Process all candidate files in `src` and its subfolders ...
         include: includeFiles,
         // When calculating the relative path to these files use this as the base path.
         // So `src/foo/bar.js` will have relative path of `foo/bar.js`
-        basePath: options.sourceDir
+        basePath: options.examplesDir
       } ];
     })
     .config(function(writeFilesProcessor) {
       // Specify where the writeFilesProcessor will write our generated doc files
-      writeFilesProcessor.outputFolder  = options.destDir;
+      writeFilesProcessor.outputFolder  = options.fragmentsDir;
     });
   return pkg;
 }
 
 var createShredMapPackage = function(mapOptions) {
-  var pkg = new Dgeni.Package('docshred-mapper', [
+  var pkg = new Dgeni.Package('doc-shred-mapper', [
     require('dgeni-packages/base'),
     require('dgeni-packages/nunjucks')
   ]);
@@ -133,7 +112,9 @@ var createShredMapPackage = function(mapOptions) {
   initializePackage(pkg)
     .factory(require('./extractPathsReader'))
     .processor(require('./shredMapProcessor'))
-
+    .config(function(shredMapProcessor) {
+      shredMapProcessor.options = options;
+    })
     .config(function(readFilesProcessor, extractPathsReader ) {
       readFilesProcessor.fileReaders = [ extractPathsReader];
     })
@@ -146,25 +127,25 @@ var createShredMapPackage = function(mapOptions) {
       var extns = ['*.jade' ];
       var includeFiles = extns.map(function(extn) {
         if (options.includeSubdirs) {
-          return path.join(options.sourceDir, '**', extn);
+          return path.join(options.jadeDir, '**', extn);
         } else {
-          return path.join(options.sourceDir, extn);
+          return path.join(options.jadeDir, extn);
         }
       });
       readFilesProcessor.sourceFiles = [ {
         // Process all candidate files in `src` and its subfolders ...
         include: includeFiles,
         // When calculating the relative path to these files use this as the base path.
         // So `src/foo/bar.js` will have relative path of `foo/bar.js`
-        basePath: options.sourceDir
+        basePath: options.jadeDir
       } ];
     })
     .config(function(writeFilesProcessor) {
       // Specify where the writeFilesProcessor will write our generated doc files
-      writeFilesProcessor.outputFolder  = options.destDir;
+      writeFilesProcessor.outputFolder  = options.outputDir;
     })
     .config(function(templateFinder) {
-      // Add a folder to search for our own templates to use when rendering docs
+      // look for templates in this folder
       templateFinder.templateFolders = [ path.resolve(__dirname) ];
 
       // Specify how to match docs to templates.
@@ -174,6 +155,7 @@ var createShredMapPackage = function(mapOptions) {
     .config(function(computePathsProcessor, computeIdsProcessor)  {
       computePathsProcessor.$enabled = false;
       computeIdsProcessor.$enabled = false;
+      // Unused for now.
       //computePathsProcessor.pathTemplates.push({
       //  docTypes: ['foo'],
       //  pathTemplate: '',
@@ -192,6 +174,30 @@ var createShredMapPackage = function(mapOptions) {
   return pkg;
 }
 
+function resolveShredOptions(shredOptions) {
+  return _.defaults({}, shredOptions, {
+    basePath: path.resolve('.'),
+    // read files from any subdir under here
+    examplesDir: "docs/_examples",
+    // shredded files get copied here with same subdir structure.
+    fragmentsDir: "docs/_fragments",
+    // whether to include subdirectories when shredding.
+    includeSubdirs: true
+  });
+}
+
+function resolveMapOptions(mapOptions) {
+  return _.defaults({}, mapOptions, {
+    basePath: path.resolve('.'),
+    // read files from any subdir under here
+    jadeDir: "docs",
+    fragmentsDir: "docs/_fragments",
+    examplesDir: "docs/_examples",
+    // whether to include subdirectories when shredding.
+    includeSubdirs: true
+  });
+}
+
 function initializePackage(pkg) {
   return pkg
     .processor(require('dgeni-packages/base/processors/read-files'))

public/doc-shredder/extractPathsReader.js

@@ -1,29 +1,30 @@
 /**
- * @dgService htmlFileShredder
+ * @dgService extractPathsReader
  * @description
  */
 
 var path = require('canonical-path');
 
 module.exports = function extractPathsReader(log) {
+  // regex for makeTabs line
   var rx = /\s*\+makeTabs\(\s*["'](.*?)["']\s*,\s*["'](.*?)["'].*?\)/g
   return {
     name: 'extractPathsReader',
 
     getDocs: function (fileInfo) {
       var content = fileInfo.content;
-      var refPaths = [];
+      var fragPaths = [];
       var r;
       while ((r = rx.exec(content)) !== null) {
         var basePath = r[1];
         var fileNames = r[2].split(',');
         fileNames.forEach(function(fn) {
-          refPaths.push(path.join(basePath, fn));
+          fragPaths.push(path.join(basePath, fn.trim()));
         })
       }
-      if (refPaths.length) {
+      if (fragPaths.length) {
         return [{
-          refPaths: refPaths
+          fragPaths: fragPaths
         }];
       } else {
         return [];

public/doc-shredder/regionExtractor.js

@@ -29,6 +29,7 @@ module.exports = function regionExtractor() {
       }
     });
 
+    var rx = new RegExp(nullLine + '\n', 'g');
     docs.forEach(function(doc) {
       var content;
       if (doc.endIx) {
@@ -37,8 +38,7 @@ module.exports = function regionExtractor() {
         content = lines.slice(doc.startIx + 1).join('\n');
       }
       // eliminate all #docregion lines
-      var rx = new RegExp(nullLine + '\n', 'g');
-      var content = content.replace(rx, '');
+      content = content.replace(rx, '');
       if (content.substr(-3) === nullLine) {
         content = content.substr(0, content.length-3);
       }

public/doc-shredder/shredMapProcessor.js

@@ -3,28 +3,53 @@
  * @description
  *
  */
+var path = require('canonical-path');
+var fs = require('fs');
+
 module.exports = function shredMapProcessor(log) {
   return {
     $runAfter: ['readFilesProcessor'],
     $runBefore: ['rendering-docs'],
+
     $process: function(docs) {
-      var docMaps = []
+      var options = this.options;
+      var jadeToFragMap = {};
+      var fragToJadeMap = {};
       docs.forEach(function(doc) {
-        var docMap = {
-          jadePath: doc.fileInfo.filePath,
-          jadeRelativePath: doc.fileInfo.projectRelativePath,
-          refPaths: doc.refPaths
-        }
-        docMaps.push(docMap);
+        var jadePath = path.join(options.jadeDir, doc.fileInfo.relativePath);
+        var fragInfos = doc.fragPaths.map(function(fragPath) {
+          fragPath =  path.join(options.fragmentsDir, fragPath) + '.md';
+          var fullPath = path.join(options.basePath, fragPath);
+          var fragInfo = { fragPath: fragPath, exists: fs.existsSync(fullPath) };
+          if (fragInfo.exists) {
+            var jadePaths = fragToJadeMap[fragInfo];
+            if (!jadePaths) {
+              jadePaths = [];
+              fragToJadeMap[fragPath] = jadePaths;
+            }
+            jadePaths.push(jadePath);
+          }
+          return fragInfo;
+        });
+        jadeToFragMap[jadePath] = fragInfos;
       });
       var newDocs = [{
-        docType: 'xref-doc.html',
-        docMaps: docMaps,
-        outputPath: 'xref-doc.html'
+        docType: 'xref-jade.html',
+        basePath: this.options.basePath,
+        jadeToFragMap: jadeToFragMap,
+        outputPath: 'xref-jade-to-frag.html'
       }, {
-        docType: 'xref-doc.js',
-        json: JSON.stringify(docMaps),
-        outputPath: 'xref-doc.js'
+        docType: 'xref-frag.html',
+        basePath: this.options.basePath,
+        fragToJadeMap: fragToJadeMap,
+        outputPath: 'xref-frag-to-jade.html'
+      }, {
+        docType: 'xref-doc.json',
+        json: JSON.stringify({
+          basePath: this.options.basePath,
+          jadeToFragMap: jadeToFragMap,
+        }, null, 2),
+        outputPath: 'xref-jade.json'
       }]
       return newDocs;
     }