1 module ddox.main;
2 
3 import ddox.ddoc;
4 import ddox.ddox;
5 import ddox.entities;
6 import ddox.htmlgenerator;
7 import ddox.htmlserver;
8 import ddox.parsers.dparse;
9 import ddox.parsers.jsonparser;
10 
11 import vibe.core.core;
12 import vibe.core.file;
13 import vibe.data.json;
14 import vibe.http.fileserver;
15 import vibe.http.router;
16 import vibe.http.server;
17 import vibe.stream.operations;
18 import std.array;
19 import std.exception : enforce;
20 import std.file;
21 import std.getopt;
22 import std.stdio;
23 import std..string;
24 
25 
26 int ddoxMain(string[] args)
27 {
28 	bool help;
29 	getopt(args, config.passThrough, "h|help", &help);
30 
31 	if( args.length < 2 || help ){
32 		showUsage(args);
33 		return help ? 0 : 1;
34 	}
35 
36 	if( args[1] == "generate-html" && args.length >= 4 )
37 		return cmdGenerateHtml(args);
38 	if( args[1] == "serve-html" && args.length >= 3 )
39 		return cmdServeHtml(args);
40 	if( args[1] == "filter" && args.length >= 3 )
41 		return cmdFilterDocs(args);
42 	if( args[1] == "serve-test" && args.length >= 3 )
43 		return cmdServeTest(args);
44 	showUsage(args);
45 	return 1;
46 }
47 
48 int cmdGenerateHtml(string[] args)
49 {
50 	GeneratorSettings gensettings;
51 	Package pack;
52 	if( auto ret = setupGeneratorInput(args, gensettings, pack) )
53 		return ret;
54 
55 	generateHtmlDocs(Path(args[3]), pack, gensettings);
56 	return 0;
57 }
58 
59 int cmdServeHtml(string[] args)
60 {
61 	string[] webfiledirs;
62 	getopt(args,
63 		config.passThrough,
64 		"web-file-dir", &webfiledirs);
65 
66 	GeneratorSettings gensettings;
67 	Package pack;
68 	if( auto ret = setupGeneratorInput(args, gensettings, pack) )
69 		return ret;
70 
71 	// register the api routes and start the server
72 	auto router = new URLRouter;
73 	registerApiDocs(router, pack, gensettings);
74 
75 	foreach (dir; webfiledirs)
76 		router.get("*", serveStaticFiles(dir));
77 
78 	writefln("Listening on port 8080...");
79 	auto settings = new HTTPServerSettings;
80 	settings.port = 8080;
81 	listenHTTP(settings, router);
82 
83 	return runEventLoop();
84 }
85 
86 int cmdServeTest(string[] args)
87 {
88 	string[] webfiledirs;
89 	auto docsettings = new DdoxSettings;
90 	auto gensettings = new GeneratorSettings;
91 
92 	auto pack = parseD(args[2 .. $]);
93 
94 	processDocs(pack, docsettings);
95 
96 	// register the api routes and start the server
97 	auto router = new URLRouter;
98 	registerApiDocs(router, pack, gensettings);
99 
100 	foreach (dir; webfiledirs)
101 		router.get("*", serveStaticFiles(dir));
102 
103 	writefln("Listening on port 8080...");
104 	auto settings = new HTTPServerSettings;
105 	settings.port = 8080;
106 	listenHTTP(settings, router);
107 
108 	return runEventLoop();
109 }
110 
111 int setupGeneratorInput(ref string[] args, out GeneratorSettings gensettings, out Package pack)
112 {
113 	string[] macrofiles;
114 	string[] overridemacrofiles;
115 	NavigationType navtype;
116 	string[] pack_order;
117 	string sitemapurl = "http://127.0.0.1/";
118 	MethodStyle file_name_style = MethodStyle.unaltered;
119 	SortMode modsort = SortMode.protectionName;
120 	SortMode declsort = SortMode.protectionInheritanceName;
121 	bool lowercasenames;
122 	bool hyphenate;
123 	bool singlepageenum;
124 	getopt(args,
125 		//config.passThrough,
126 		"decl-sort", &declsort,
127 		"file-name-style", &file_name_style,
128 		"hyphenate", &hyphenate,
129 		"lowercase-names", &lowercasenames,
130 		"module-sort", &modsort,
131 		"navigation-type", &navtype,
132 		"override-macros", &overridemacrofiles,
133 		"package-order", &pack_order,
134 		"sitemap-url", &sitemapurl,
135 		"std-macros", &macrofiles,
136 		"enum-member-pages", &singlepageenum,
137 		);
138 
139 	if (lowercasenames) file_name_style = MethodStyle.lowerCase;
140 
141 	if( args.length < 3 ){
142 		showUsage(args);
143 		return 1;
144 	}
145 
146 	setDefaultDdocMacroFiles(macrofiles);
147 	setOverrideDdocMacroFiles(overridemacrofiles);
148 	if (hyphenate) enableHyphenation();
149 
150 	// parse the json output file
151 	auto docsettings = new DdoxSettings;
152 	docsettings.packageOrder = pack_order;
153 	docsettings.moduleSort = modsort;
154 	docsettings.declSort = declsort;
155 	pack = parseDocFile(args[2], docsettings);
156 
157 	gensettings = new GeneratorSettings;
158 	gensettings.siteUrl = URL(sitemapurl);
159 	gensettings.navigationType = navtype;
160 	gensettings.fileNameStyle = file_name_style;
161 	gensettings.enumMemberPages = singlepageenum;
162 	return 0;
163 }
164 
165 int cmdFilterDocs(string[] args)
166 {
167 	string[] excluded, included;
168 	Protection minprot = Protection.Private;
169 	bool keeputests = false;
170 	bool keepinternals = false;
171 	bool unittestexamples = true;
172 	bool nounittestexamples = false;
173 	bool justdoc = false;
174 	getopt(args,
175 		//config.passThrough,
176 		"ex", &excluded,
177 		"in", &included,
178 		"min-protection", &minprot,
179 		"only-documented", &justdoc,
180 		"keep-unittests", &keeputests,
181 		"keep-internals", &keepinternals,
182 		"unittest-examples", &unittestexamples, // deprecated, kept to not break existing scripts
183 		"no-unittest-examples", &nounittestexamples);
184 
185 	if (keeputests) keepinternals = true;
186 	if (nounittestexamples) unittestexamples = false;
187 
188 	string jsonfile;
189 	if( args.length < 3 ){
190 		showUsage(args);
191 		return 1;
192 	}
193 
194 	Json filterProt(Json json, Json parent, Json last_decl, Json mod)
195 	{
196 		if (last_decl.type == Json.Type.undefined) last_decl = parent;
197 
198 		string templateName(Json j){
199 			auto n = j.name.opt!string();
200 			auto idx = n.indexOf('(');
201 			if( idx >= 0 ) return n[0 .. idx];
202 			return n;
203 		}
204 
205 		if( json.type == Json.Type.Object ){
206 			auto comment = json.comment.opt!string;
207 			if( justdoc && comment.empty ){
208 				if( parent.type != Json.Type.Object || parent.kind.opt!string() != "template" || templateName(parent) != json.name.opt!string() )
209 					return Json.undefined;
210 			}
211 
212 			Protection prot = Protection.Public;
213 			if( auto p = "protection" in json ){
214 				switch(p.get!string){
215 					default: break;
216 					case "private": prot = Protection.Private; break;
217 					case "package": prot = Protection.Package; break;
218 					case "protected": prot = Protection.Protected; break;
219 				}
220 			}
221 			if( comment.strip == "private" ) prot = Protection.Private;
222 			if( prot < minprot ) return Json.undefined;
223 
224 			auto name = json.name.opt!string();
225 			bool is_internal = name.startsWith("__");
226 			bool is_unittest = name.startsWith("__unittest");
227 			if (name.startsWith("_staticCtor") || name.startsWith("_staticDtor")) is_internal = true;
228 			else if (name.startsWith("_sharedStaticCtor") || name.startsWith("_sharedStaticDtor")) is_internal = true;
229 
230 			if (unittestexamples && is_unittest && !comment.empty) {
231 				assert(last_decl.type == Json.Type.object, "Don't have a last_decl context.");
232 				try {
233 					string source = extractUnittestSourceCode(json, mod);
234 					if (last_decl.comment.opt!string.empty) {
235 						writefln("Warning: Cannot add documented unit test %s to %s, which is not documented.", name, last_decl.name.opt!string);
236 					} else {
237 						last_decl.comment ~= format("Example:\n%s\n---\n%s\n---\n", comment.strip, source);
238 					}
239 				} catch (Exception e) {
240 					writefln("Failed to add documented unit test %s:%s as example: %s",
241 						mod.file.get!string(), json["line"].get!long, e.msg);
242 					return Json.undefined;
243 				}
244 			}
245 
246 			if (!keepinternals && is_internal) return Json.undefined;
247 
248 			if (!keeputests && is_unittest) return Json.undefined;
249 
250 			if (auto mem = "members" in json)
251 				json.members = filterProt(*mem, json, Json.undefined, mod);
252 		} else if( json.type == Json.Type.Array ){
253 			auto last_child_decl = Json.undefined;
254 			Json[] newmem;
255 			foreach (m; json) {
256 				auto mf = filterProt(m, parent, last_child_decl, mod);
257 				if (mf.type == Json.Type.undefined) continue;
258 				if (mf.type == Json.Type.object && !mf.name.opt!string.startsWith("__unittest") && icmp(mf.comment.opt!string.strip, "ditto") != 0)
259 					last_child_decl = mf;
260 				newmem ~= mf;
261 			}
262 			return Json(newmem);
263 		}
264 		return json;
265 	}
266 
267 	writefln("Reading doc file...");
268 	auto text = readText(args[2]);
269 	int line = 1;
270 	writefln("Parsing JSON...");
271 	auto json = parseJson(text, &line);
272 
273 	writefln("Filtering modules...");
274 	Json[] dst;
275 	foreach (m; json) {
276 		if ("name" !in m) {
277 			writefln("No name for module %s - ignoring", m.file.opt!string);
278 			continue;
279 		}
280 		auto n = m.name.get!string;
281 		bool include = true;
282 		foreach (ex; excluded)
283 			if (n.startsWith(ex)) {
284 				include = false;
285 				break;
286 			}
287 		foreach (inc; included)
288 			if (n.startsWith(inc)) {
289 				include = true;
290 				break;
291 			}
292 		if (include) {
293 			auto doc = filterProt(m, Json.undefined, Json.undefined, m);
294 			if (doc.type != Json.Type.undefined)
295 				dst ~= doc;
296                 }
297 	}
298 
299 	writefln("Writing filtered docs...");
300 	auto buf = appender!string();
301 	writePrettyJsonString(buf, Json(dst));
302 	std.file.write(args[2], buf.data());
303 
304 	return 0;
305 }
306 
307 Package parseDocFile(string filename, DdoxSettings settings)
308 {
309 	writefln("Reading doc file...");
310 	auto text = readText(filename);
311 	int line = 1;
312 	writefln("Parsing JSON...");
313 	auto json = parseJson(text, &line);
314 	writefln("Parsing docs...");
315 	Package root;
316 	root = parseJsonDocs(json);
317 	writefln("Finished parsing docs.");
318 
319 	processDocs(root, settings);
320 	return root;
321 }
322 
323 void showUsage(string[] args)
324 {
325 	string cmd;
326 	if( args.length >= 2 ) cmd = args[1];
327 
328 	switch(cmd){
329 		default:
330 			writefln(
331 `Usage: %s <COMMAND> [args...]
332 
333     <COMMAND> can be one of:
334         generate-html
335         serve-html
336         filter
337 
338  -h --help                 Show this help
339 
340 Use <COMMAND> -h|--help to get detailed usage information for a command.
341 `, args[0]);
342 			break;
343 		case "serve-html":
344 			writefln(
345 `Usage: %s serve-html <ddocx-input-file>
346     --std-macros=FILE      File containing DDOC macros that will be available
347     --override-macros=FILE File containing DDOC macros that will override local
348                            definitions (Macros: section)
349     --navigation-type=TYPE Change the type of navigation (ModuleList,
350                            ModuleTree, DeclarationTree)
351     --package-order=NAME   Causes the specified module to be ordered first. Can
352                            be specified multiple times.
353     --sitemap-url          Specifies the base URL used for sitemap generation
354     --module-sort=MODE     The sort order used for lists of modules
355     --decl-sort=MODE       The sort order used for declaration lists
356     --web-file-dir=DIR     Make files from dir available on the served site
357     --enum-member-pages    Generate a single page per enum member
358     --hyphenate            hyphenate text
359  -h --help                 Show this help
360 
361 The following values can be used as sorting modes: none, name, protectionName,
362 protectionInheritanceName
363 `, args[0]);
364 			break;
365 		case "generate-html":
366 			writefln(
367 `Usage: %s generate-html <ddocx-input-file> <output-dir>
368     --std-macros=FILE      File containing DDOC macros that will be available
369     --override-macros=FILE File containing DDOC macros that will override local
370                            definitions (Macros: section)
371     --navigation-type=TYPE Change the type of navigation (ModuleList,
372                            ModuleTree, DeclarationTree)
373     --package-order=NAME   Causes the specified module to be ordered first. Can
374                            be specified multiple times.
375     --sitemap-url          Specifies the base URL used for sitemap generation
376     --module-sort=MODE     The sort order used for lists of modules
377     --decl-sort=MODE       The sort order used for declaration lists
378     --file-name-style=STY  Sets a translation style for symbol names to file
379                            names. Use this instead of --lowercase-name.
380                            Possible values for STY:
381                              unaltered, camelCase, pascalCase, lowerCase,
382                              upperCase, lowerUnderscored, upperUnderscored
383     --lowercase-names      DEPRECATED: Outputs all file names in lower case.
384                            This option is useful on case insensitive file
385                            systems.
386     --enum-member-pages    Generate a single page per enum member
387     --hyphenate            hyphenate text
388  -h --help                 Show this help
389 
390 The following values can be used as sorting modes: none, name, protectionName,
391 protectionInheritanceName
392 `, args[0]);
393 			break;
394 		case "filter":
395 			writefln(
396 `Usage: %s filter <ddocx-input-file> [options]
397     --ex=PREFIX            Exclude modules with prefix
398     --in=PREFIX            Force include of modules with prefix
399     --min-protection=PROT  Remove items with lower protection level than
400                            specified.
401                            PROT can be: Public, Protected, Package, Private
402     --only-documented      Remove undocumented entities.
403     --keep-unittests       Do not remove unit tests from documentation.
404                            Implies --keep-internals.
405     --keep-internals       Do not remove symbols starting with two underscores.
406     --unittest-examples    Add documented unit tests as examples to the
407                            preceding declaration (deprecated, enabled by
408                            default)
409     --no-unittest-examples Don't convert documented unit tests to examples
410  -h --help                 Show this help
411 `, args[0]);
412 	}
413 	if( args.length < 2 ){
414 	} else {
415 
416 	}
417 }
418 
419 private string extractUnittestSourceCode(Json decl, Json mod)
420 {
421 	auto filename = mod.file.get!string();
422 	enforce("line" in decl && "endline" in decl, "Missing line/endline fields.");
423 	auto from = decl["line"].get!long;
424 	auto to = decl.endline.get!long;
425 
426 	// read the matching lines out of the file
427 	auto app = appender!string();
428 	long lc = 1;
429 	foreach (str; File(filename).byLine) {
430 		if (lc >= from) {
431 			app.put(str);
432 			app.put('\n');
433 		}
434 		if (++lc > to) break;
435 	}
436 	auto ret = app.data;
437 
438 	// strip the "unittest { .. }" surroundings
439 	auto idx = ret.indexOf("unittest");
440 	enforce(idx >= 0, format("Missing 'unittest' for unit test at %s:%s.", filename, from));
441 	ret = ret[idx .. $];
442 
443 	idx = ret.indexOf("{");
444 	enforce(idx >= 0, format("Missing opening '{' for unit test at %s:%s.", filename, from));
445 	ret = ret[idx+1 .. $];
446 
447 	idx = ret.lastIndexOf("}");
448 	enforce(idx >= 0, format("Missing closing '}' for unit test at %s:%s.", filename, from));
449 	ret = ret[0 .. idx];
450 
451 	// unindent lines according to the indentation of the first line
452 	app = appender!string();
453 	string indent;
454 	foreach (i, ln; ret.splitLines) {
455 		if (i == 1) {
456 			foreach (j; 0 .. ln.length)
457 				if (ln[j] != ' ' && ln[j] != '\t') {
458 					indent = ln[0 .. j];
459 					break;
460 				}
461 		}
462 		if (i > 0 || ln.strip.length > 0) {
463 			size_t j = 0;
464 			while (j < indent.length && !ln.empty) {
465 				if (ln.front != indent[j]) break;
466 				ln.popFront();
467 				j++;
468 			}
469 			app.put(ln);
470 			app.put('\n');
471 		}
472 	}
473 	return app.data;
474 }