refactor(gen-docs): use q, qq, q-fs (node modules) to write gen-docs

- re-write gendocs.js, reader.js and writer.js
- all calls are asynchronous
This commit is contained in:
Di Peng 2011-07-09 18:15:40 -07:00 committed by Igor Minar
parent e90b741c94
commit 8fa066190a
8 changed files with 308 additions and 347 deletions

1
.gitignore vendored
View file

@ -9,3 +9,4 @@ performance/temp*.html
.idea/workspace.xml
*~
angular.js.tmproj
node_modules

View file

@ -246,7 +246,7 @@ task :package => [:clean, :compile, :docs] do
f.write text.sub('angular-scenario.js', "angular-scenario-#{version}.js")
end
File.open("#{pkg_dir}/docs-#{version}/app-cache.manifest", File::RDWR) do |f|
File.open("#{pkg_dir}/docs-#{version}/appcache.manifest", File::RDWR) do |f|
text = f.read
f.rewind
f.write text.sub('angular.min.js', "angular-#{version}.min.js")

View file

@ -3,48 +3,73 @@
*/
exports.appCache = appCache;
var fs = require('fs');
var fs = require('q-fs');
var Q = require('qq');
function identity($) {return $;}
function appCache(path) {
var blackList = [ "offline.html",
"sitemap.xml",
"robots.txt",
"docs-scenario.html",
"docs-scenario.js",
"app-cache.manifest"
if(!path) {
return appCacheTemplate();
}
var blackList = ["offline.html",
"sitemap.xml",
"robots.txt",
"docs-scenario.html",
"docs-scenario.js",
"appcache.manifest"
];
var result = ["CACHE MANIFEST",
"# %TIMESTAMP%",
"",
"# cache all of these",
"CACHE:",
"../angular.min.js"];
"# " + new Date().toISOString(),
"",
"# cache all of these",
"CACHE:",
"../angular.min.js"];
var resultPostfix = [ "",
"FALLBACK:",
"/offline.html",
"",
"# allow access to google analytics and twitter when we are online",
"NETWORK:",
"*"];
walk(path,result,blackList);
return result.join('\n').replace(/%TIMESTAMP%/, (new Date()).toISOString()) + '\n' + resultPostfix.join('\n');
var resultPostfix = ["",
"FALLBACK:",
"/offline.html",
"",
"# allow access to google analytics and twitter when we are online",
"NETWORK:",
"*"];
var promise = fs.listTree(path).then(function(files){
var fileFutures = [];
files.forEach(function(file){
fileFutures.push(fs.isFile(file).then(function(isFile){
if (isFile && blackList.indexOf(file) == -1) {
return file.replace('build/docs/','');
}
}));
});
return Q.deep(fileFutures);
}).then(function(files){
return result.concat(files.filter(identity)).concat(resultPostfix).join('\n');
});
return promise;
}
function walk(path, array, blackList) {
var temp = fs.readdirSync(path);
for (var i=0; i< temp.length; i++) {
if(blackList.indexOf(temp[i]) < 0) {
var currentPath = path + '/' + temp[i];
var stat = fs.statSync(currentPath);
if (stat.isDirectory()) {
walk(currentPath, array, blackList);
}
else {
array.push(currentPath.replace('build/docs/',''));
}
}
}
}
function appCacheTemplate() {
return ["CACHE MANIFEST",
"# " + new Date().toISOString(),
"",
"# cache all of these",
"CACHE:",
"syntaxhighlighter/syntaxhighlighter-combined.js",
"../angular.min.js",
"docs-combined.js",
"docs-keywords.js",
"docs-combined.css",
"syntaxhighlighter/syntaxhighlighter-combined.css",
"img/texture_1.png",
"img/yellow_bkgnd.jpg",
"",
"FALLBACK:",
"/ offline.html",
"",
"# allow access to google analytics and twitter when we are online",
"NETWORK:",
"*"].join('\n');
}

View file

@ -1,69 +0,0 @@
function noop(){}
function chain(delegateFn, explicitDone){
var onDoneFn = noop;
var onErrorFn = function(e){
console.error(e.stack || e);
process.exit(-1);
};
var waitForCount = 1;
delegateFn = delegateFn || noop;
var stackError = new Error('capture stack');
function decrementWaitFor() {
waitForCount--;
if (waitForCount == 0)
onDoneFn();
}
function self(){
try {
return delegateFn.apply(self, arguments);
} catch (error) {
self.error(error);
} finally {
if (!explicitDone)
decrementWaitFor();
}
};
self.onDone = function(callback){
onDoneFn = callback;
return self;
};
self.onError = function(callback){
onErrorFn = callback;
return self;
};
self.waitFor = function(callback){
if (waitForCount == 0)
throw new Error("Can not wait on already called callback.");
waitForCount++;
return chain(callback).onDone(decrementWaitFor).onError(self.error);
};
self.waitMany = function(callback){
if (waitForCount == 0)
throw new Error("Can not wait on already called callback.");
waitForCount++;
return chain(callback, true).onDone(decrementWaitFor).onError(self.error);
};
self.done = function(callback){
decrementWaitFor();
};
self.error = function(error) {
var stack = stackError.stack.split(/\n\r?/).splice(2);
var nakedStack = [];
stack.forEach(function(frame){
if (!frame.match(/callback\.js:\d+:\d+\)$/))
nakedStack.push(frame);
});
error.stack = error.stack + '\nCalled from:\n' + nakedStack.join('\n');
onErrorFn(error);
};
return self;
}
exports.chain = chain;

View file

@ -3,90 +3,77 @@ require.paths.push('lib');
var reader = require('reader.js'),
ngdoc = require('ngdoc.js'),
writer = require('writer.js'),
callback = require('callback.js'),
SiteMap = require('SiteMap.js').SiteMap,
appCache = require('appCache.js');
appCache = require('appCache.js').appCache,
Q = require('qq');
var docs = [];
var start;
var work = callback.chain(function(){
start = now();
process.on('uncaughtException', function (err) {
console.error(err.stack || err);
});
var start = now();
var docs;
writer.makeDir('build/docs/syntaxhighlighter').then(function() {
console.log('Generating Angular Reference Documentation...');
reader.collect(work.waitMany(function(text, file, line){
var doc = new ngdoc.Doc(text, file, line);
docs.push(doc);
doc.parse();
}));
});
var writes = callback.chain(function(){
return reader.collect();
}).then(function generateHtmlDocPartials(docs_) {
docs = docs_;
ngdoc.merge(docs);
var fileFutures = [];
docs.forEach(function(doc){
writer.output(doc.section + '/' + doc.id + '.html', doc.html(), writes.waitFor());
fileFutures.push(writer.output(doc.section + '/' + doc.id + '.html', doc.html()));
});
var metadata = ngdoc.metadata(docs);
writer.output('docs-keywords.js', ['NG_PAGES=', JSON.stringify(metadata).replace(/{/g, '\n{'), ';'], writes.waitFor());
writer.copyDir('img', writes.waitFor());
writer.copyDir('examples', writes.waitFor());
writer.copyTpl('index.html', writes.waitFor());
writer.copyTpl('.htaccess', writes.waitFor());
writer.copy('docs/src/templates/index.html', 'build/docs/index-jq.html', writes.waitFor(),
'<-- jquery place holder -->', '<script src=\"jquery.min.js\"><\/script>');
writer.copyTpl('offline.html', writes.waitFor());
//writer.output('app-cache.manifest',
// appCacheTemplate().replace(/%TIMESTAMP%/, (new Date()).toISOString()),
// writes.waitFor());
writer.merge(['docs.js',
'doc_widgets.js'],
'docs-combined.js',
writes.waitFor());
writer.merge(['docs.css',
'doc_widgets.css'],
'docs-combined.css',
writes.waitFor());
writer.copyTpl('docs-scenario.html', writes.waitFor());
writer.output('docs-scenario.js', ngdoc.scenarios(docs), writes.waitFor());
writer.output('sitemap.xml', new SiteMap(docs).render(), writes.waitFor());
writer.output('robots.txt', 'Sitemap: http://docs.angularjs.org/sitemap.xml\n', writes.waitFor());
writer.merge(['syntaxhighlighter/shCore.js',
'syntaxhighlighter/shBrushJScript.js',
'syntaxhighlighter/shBrushXml.js'],
'syntaxhighlighter/syntaxhighlighter-combined.js',
writes.waitFor());
writer.merge(['syntaxhighlighter/shCore.css',
'syntaxhighlighter/shThemeDefault.css'],
'syntaxhighlighter/syntaxhighlighter-combined.css',
writes.waitFor());
writer.copyTpl('jquery.min.js', writes.waitFor());
writer.output('app-cache.manifest', appCache('build/docs/'), writes.waitFor());
});
writes.onDone(function(){
console.log('DONE. Generated ' + docs.length + ' pages in ' +
(now()-start) + 'ms.' );
});
work.onDone(writes);
writer.makeDir('build/docs/syntaxhighlighter', work);
///////////////////////////////////
writeTheRest(fileFutures);
return Q.deep(fileFutures);
}).then(function generateManifestFile() {
return appCache('build/docs/').then(function(list) {
writer.output('appcache-offline.manifest',list)
});
}).then(function printStats() {
console.log('DONE. Generated ' + docs.length + ' pages in ' + (now()-start) + 'ms.' );
}).end();
function writeTheRest(writesFuture) {
var metadata = ngdoc.metadata(docs);
writesFuture.push(writer.copyDir('img'));
writesFuture.push(writer.copyDir('examples'));
writesFuture.push(writer.copyTpl('index.html'));
writesFuture.push(writer.copy('docs/src/templates/index.html',
'build/docs/index-jq.html',
'<!-- jquery place holder -->',
'<script src=\"jquery.min.js\"><\/script>'));
writesFuture.push(writer.copyTpl('offline.html'));
writesFuture.push(writer.copyTpl('docs-scenario.html'));
writesFuture.push(writer.copyTpl('jquery.min.js'));
writesFuture.push(writer.output('docs-keywords.js',
['NG_PAGES=', JSON.stringify(metadata).replace(/{/g, '\n{'), ';']));
writesFuture.push(writer.output('sitemap.xml', new SiteMap(docs).render()));
writesFuture.push(writer.output('docs-scenario.js', ngdoc.scenarios(docs)));
writesFuture.push(writer.output('robots.txt', 'Sitemap: http://docs.angularjs.org/sitemap.xml\n'));
writesFuture.push(writer.output('appcache.manifest',appCache()));
writesFuture.push(writer.merge(['docs.js',
'doc_widgets.js'],
'docs-combined.js'));
writesFuture.push(writer.merge(['docs.css',
'doc_widgets.css'],
'docs-combined.css'));
writesFuture.push(writer.merge(['syntaxhighlighter/shCore.js',
'syntaxhighlighter/shBrushJScript.js',
'syntaxhighlighter/shBrushXml.js'],
'syntaxhighlighter/syntaxhighlighter-combined.js'));
writesFuture.push(writer.merge(['syntaxhighlighter/shCore.css',
'syntaxhighlighter/shThemeDefault.css'],
'syntaxhighlighter/syntaxhighlighter-combined.css'));
}
function now(){ return new Date().getTime(); }
function appCacheTemplate() {
return ["CACHE MANIFEST",
"# %TIMESTAMP%",
"",
"# cache all of these",
"CACHE:",
"syntaxhighlighter/syntaxhighlighter-combined.js",
"../angular.min.js",
"docs-combined.js",
"docs-keywords.js",
"docs-combined.css",
"syntaxhighlighter/syntaxhighlighter-combined.css",
"",
"FALLBACK:",
"/ offline.html",
"",
"# allow access to google analytics and twitter when we are online",
"NETWORK:",
"*"].join('\n');
}
function noop(){};

View file

@ -2,98 +2,98 @@
* All reading related code here. This is so that we can separate the async code from sync code
* for testability
*/
exports.collect = collect;
require.paths.push(__dirname);
var fs = require('fs'),
callback = require('callback');
var ngdoc = require('ngdoc.js'),
Q = require('qq'),
qfs = require('q-fs');
var NEW_LINE = /\n\r?/;
function collect(callback){
findJsFiles('src', callback.waitMany(function(file) {
console.log('reading', file, '...');
findNgDocInJsFile(file, callback.waitMany(function(doc, line) {
callback('@section api\n' + doc, file, line);
}));
}));
findNgDocInDir('docs/content', callback.waitMany(callback));
callback.done();
}
function collect() {
var allDocs = [];
function findJsFiles(dir, callback){
fs.readdir(dir, callback.waitFor(function(err, files){
if (err) return this.error(err);
files.forEach(function(file){
var path = dir + '/' + file;
fs.lstat(path, callback.waitFor(function(err, stat){
if (err) return this.error(err);
if (stat.isDirectory())
findJsFiles(path, callback.waitMany(callback));
else if (/\.js$/.test(path))
callback(path);
}));
});
callback.done();
}));
}
function findNgDocInDir(directory, docNotify) {
fs.readdir(directory, docNotify.waitFor(function(err, files){
if (err) return this.error(err);
files.forEach(function(file){
fs.stat(directory + '/' + file, docNotify.waitFor(function(err, stats){
if (err) return this.error(err);
if (stats.isFile()) {
if (!file.match(/\.ngdoc$/)) return;
console.log('reading', directory + '/' + file, '...');
fs.readFile(directory + '/' + file, docNotify.waitFor(function(err, content){
if (err) return this.error(err);
var section = '@section ' + directory.split('/').pop() + '\n';
docNotify(section + content.toString(), directory + '/' +file, 1);
}));
} else if(stats.isDirectory()) {
findNgDocInDir(directory + '/' + file, docNotify.waitFor(docNotify));
}
}));
});
docNotify.done();
}));
}
function findNgDocInJsFile(file, callback) {
fs.readFile(file, callback.waitFor(function(err, content){
var lines = content.toString().split(NEW_LINE);
var text;
var startingLine ;
var match;
var inDoc = false;
lines.forEach(function(line, lineNumber){
lineNumber++;
// is the comment starting?
if (!inDoc && (match = line.match(/^\s*\/\*\*\s*(.*)$/))) {
line = match[1];
inDoc = true;
text = [];
startingLine = lineNumber;
}
// are we done?
if (inDoc && line.match(/\*\//)) {
text = text.join('\n');
text = text.replace(/^\n/, '');
if (text.match(/@ngdoc/)){
callback(text, startingLine);
}
doc = null;
inDoc = false;
}
// is the comment add text
if (inDoc){
text.push(line.replace(/^\s*\*\s?/, ''));
//collect docs in JS Files
var path = 'src';
var promiseA = Q.when(qfs.listTree(path), function(files) {
var done;
//read all files in parallel.
files.forEach(function(file) {
var work;
if(/\.js$/.test(file)) {
console.log("reading " + file + ".......");
work = Q.when(qfs.read(file), function(content) {
processJsFile(content, file).forEach (function(doc) {
allDocs.push(doc);
});
});
}
done = Q.when(done, function() {
return work;
});
});
callback.done();
}));
return done;
});
//collect all NG Docs in Content Folder
var path2 = 'docs/content';
var promiseB = Q.when(qfs.listTree(path2), function(files){
var done2;
files.forEach(function(file) {
var work2;
if (file.match(/\.ngdoc$/)) {
console.log("reading " + file + ".......");
work2 = Q.when(qfs.read(file), function(content){
var section = '@section ' + file.split('/')[2] + '\n';
allDocs.push(new ngdoc.Doc(section + content.toString(),file, 1).parse());
});
}
done2 = Q.when(done2, function() {
return work2;
});
});
return done2;
});
return Q.join(promiseA, promiseB, function() {
return allDocs;
});
}
function processJsFile(content, file) {
var docs = [];
var lines = content.toString().split(NEW_LINE);
var text;
var startingLine ;
var match;
var inDoc = false;
exports.collect = collect;
lines.forEach(function(line, lineNumber){
lineNumber++;
// is the comment starting?
if (!inDoc && (match = line.match(/^\s*\/\*\*\s*(.*)$/))) {
line = match[1];
inDoc = true;
text = [];
startingLine = lineNumber;
}
// are we done?
if (inDoc && line.match(/\*\//)) {
text = text.join('\n');
text = text.replace(/^\n/, '');
if (text.match(/@ngdoc/)){
//console.log(file, startingLine)
docs.push(new ngdoc.Doc('@section api\n' + text, file, startingLine).parse());
}
doc = null;
inDoc = false;
}
// is the comment add text
if (inDoc){
text.push(line.replace(/^\s*\*\s?/, ''));
}
});
return docs;
}

View file

@ -2,7 +2,7 @@
<html xmlns:ng="http://angularjs.org/"
xmlns:doc="http://docs.angularjs.org/"
ng:controller="DocsController"
manifest="app-cache.manifest">
manifest="appcache.manifest">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title ng:bind-template="AngularJS: {{partialTitle}}">AngularJS</title>
@ -76,7 +76,7 @@
</div>
<script src="syntaxhighlighter/syntaxhighlighter-combined.js"></script>
<-- jquery place holder -->
<!-- jquery place holder -->
<script src="../angular.min.js" ng:autobind></script>
<script src="docs-combined.js"></script>
<script src="docs-keywords.js"></script>

View file

@ -3,37 +3,113 @@
* for testability
*/
require.paths.push(__dirname);
var fs = require('fs');
var qfs = require('q-fs');
var Q = require('qq');
var OUTPUT_DIR = "build/docs/";
var fs = require('fs');
function output(docs, content, callback){
callback();
exports.output = function(file, content){
console.log('writing ', file);
var fullPath = OUTPUT_DIR + file;
var dir = parent(fullPath);
return Q.when(exports.makeDir(dir), function(error) {
qfs.write(fullPath,exports.toString(content));
});
}
//recursively create directory
exports.makeDir = function (path) {
var parts = path.split(/\//);
var path = ".";
//Sequentially create directories
var done = Q.defer();
(function createPart() {
if(!parts.length) {
done.resolve();
} else {
path += "/" + parts.shift();
qfs.isDirectory(path).then(function(isDir) {
if(!isDir) {
qfs.makeDirectory(path);
}
createPart();
});
}
})();
return done.promise;
};
exports.copyTpl = function(filename) {
return exports.copy('docs/src/templates/' + filename, OUTPUT_DIR + filename);
};
exports.copy = function (from, to, replacementKey, replacement) {
// Have to use rb (read binary), char 'r' is infered by library.
return qfs.read(from,'b').then(function(content) {
if(replacementKey && replacement) {
content = content.toString().replace(replacementKey, replacement);
}
qfs.write(to, content);
});
}
exports.copyDir = function copyDir(dir) {
return qfs.listDirectoryTree('docs/' + dir).then(function(dirs) {
var done;
dirs.forEach(function(dirToMake) {
done = Q.when(done, function() {
return exports.makeDir("./build/" + dirToMake);
});
});
return done;
}).then(function() {
return qfs.listTree('docs/' + dir);
}).then(function(files) {
files.forEach( function(file) {
exports.copy(file,'./build/' + file);
});
});
};
exports.merge = function(srcs, to) {
return merge(srcs.map(function(src) { return 'docs/src/templates/' + src; }), OUTPUT_DIR + to);
};
function merge(srcs, to) {
var contents = [];
//Sequentially read file
var done;
srcs.forEach(function (src) {
done = Q.when(done, function(content) {
if(content) contents.push(content);
return qfs.read(src);
});
});
// write to file
return Q.when(done, function(content) {
contents.push(content);
qfs.write(to, contents.join('\n'));
});
}
//----------------------- Synchronous Methods ----------------------------------
function parent(file) {
var parts = file.split('/');
parts.pop();
return parts.join('/');
}
exports.output = function(file, content, callback){
console.log('write', file);
exports.makeDir(parent(OUTPUT_DIR + file), callback.waitFor(function(){
fs.writeFile(
OUTPUT_DIR + file,
exports.toString(content),
callback);
}));
};
exports.toString = function toString(obj){
exports.toString = function toString(obj) {
switch (typeof obj) {
case 'string':
return obj;
case 'object':
if (obj instanceof Array) {
obj.forEach(function (value, key){
obj.forEach(function (value, key) {
obj[key] = toString(value);
});
return obj.join('');
@ -44,64 +120,5 @@ exports.toString = function toString(obj){
return obj;
};
exports.makeDir = function (path, callback) {
var parts = path.split(/\//);
path = '.';
(function next(error){
if (error && error.code != 'EEXIST') return callback.error(error);
if (parts.length) {
path += '/' + parts.shift();
fs.mkdir(path, 0777, next);
} else {
callback();
}
})();
};
exports.copyTpl = function(filename, callback) {
exports.copy('docs/src/templates/' + filename, OUTPUT_DIR + filename, callback);
};
exports.copy = function(from, to, callback, replacementKey, replacement) {
//console.log('writing', to, '...');
fs.readFile(from, function(err, content){
if (err) return callback.error(err);
if(replacementKey && replacement) {
content = content.toString().replace(replacementKey, replacement);
}
fs.writeFile(to, content, callback);
});
};
exports.copyDir = function copyDir(dir, callback) {
exports.makeDir(OUTPUT_DIR + '/' + dir, callback.waitFor(function(){
fs.readdir('docs/' + dir, callback.waitFor(function(err, files){
if (err) return this.error(err);
files.forEach(function(file){
var path = 'docs/' + dir + '/' + file;
fs.stat(path, callback.waitFor(function(err, stat) {
if (err) return this.error(err);
if (stat.isDirectory()) {
copyDir(dir + '/' + file, callback.waitFor());
} else {
exports.copy(path, OUTPUT_DIR + '/' + dir + '/' + file, callback.waitFor());
}
}));
});
callback();
}));
}));
};
exports.merge = function(srcs, to, callback){
merge(srcs.map(function(src) { return 'docs/src/templates/' + src; }), OUTPUT_DIR + to, callback);
};
function merge(srcs, to, callback) {
var content = [];
srcs.forEach(function (src) {
content.push(fs.readFileSync(src));
});
fs.writeFile(to, content.join('\n'), callback.waitFor());
}
function noop(){};