1 module ddox.main;
2 
3 import ddox.ddoc;
4 import ddox.ddox;
5 import ddox.entities;
6 import ddox.htmlgenerator;
7 import ddox.htmlserver;
8 import ddox.parsers.dparse;
9 import ddox.parsers.jsonparser;
10 
11 import vibe.core.core;
12 import vibe.core.file;
13 import vibe.data.json;
14 import vibe.inet.url;
15 import vibe.http.fileserver;
16 import vibe.http.router;
17 import vibe.http.server;
18 import vibe.stream.operations;
19 import std.array;
20 import std.exception : enforce;
21 import std.file;
22 import std.getopt;
23 import std.stdio;
24 import std..string;
25 
26 
27 int ddoxMain(string[] args)
28 {
29 	bool help;
30 	getopt(args, config.passThrough, "h|help", &help);
31 
32 	if( args.length < 2 || help ){
33 		showUsage(args);
34 		return help ? 0 : 1;
35 	}
36 
37 	if( args[1] == "generate-html" && args.length >= 4 )
38 		return cmdGenerateHtml(args);
39 	if( args[1] == "serve-html" && args.length >= 3 )
40 		return cmdServeHtml(args);
41 	if( args[1] == "filter" && args.length >= 3 )
42 		return cmdFilterDocs(args);
43 	if( args[1] == "serve-test" && args.length >= 3 )
44 		return cmdServeTest(args);
45 	showUsage(args);
46 	return 1;
47 }
48 
49 int cmdGenerateHtml(string[] args)
50 {
51 	GeneratorSettings gensettings;
52 	Package pack;
53 	if( auto ret = setupGeneratorInput(args, gensettings, pack) )
54 		return ret;
55 
56 	generateHtmlDocs(NativePath(args[3]), pack, gensettings);
57 	return 0;
58 }
59 
60 int cmdServeHtml(string[] args)
61 {
62 	string[] webfiledirs;
63 	getopt(args,
64 		config.passThrough,
65 		"web-file-dir", &webfiledirs);
66 
67 	GeneratorSettings gensettings;
68 	Package pack;
69 	if( auto ret = setupGeneratorInput(args, gensettings, pack) )
70 		return ret;
71 
72 	// register the api routes and start the server
73 	auto router = new URLRouter;
74 	registerApiDocs(router, pack, gensettings);
75 
76 	foreach (dir; webfiledirs)
77 		router.get("*", serveStaticFiles(dir));
78 
79 	writefln("Listening on port 8080...");
80 	auto settings = new HTTPServerSettings;
81 	settings.port = 8080;
82 	listenHTTP(settings, router);
83 
84 	return runEventLoop();
85 }
86 
87 int cmdServeTest(string[] args)
88 {
89 	string[] webfiledirs;
90 	auto docsettings = new DdoxSettings;
91 	auto gensettings = new GeneratorSettings;
92 
93 	auto pack = parseD(args[2 .. $]);
94 
95 	processDocs(pack, docsettings);
96 
97 	// register the api routes and start the server
98 	auto router = new URLRouter;
99 	registerApiDocs(router, pack, gensettings);
100 
101 	foreach (dir; webfiledirs)
102 		router.get("*", serveStaticFiles(dir));
103 
104 	writefln("Listening on port 8080...");
105 	auto settings = new HTTPServerSettings;
106 	settings.port = 8080;
107 	listenHTTP(settings, router);
108 
109 	return runEventLoop();
110 }
111 
112 int setupGeneratorInput(ref string[] args, out GeneratorSettings gensettings, out Package pack)
113 {
114 	gensettings = new GeneratorSettings;
115 	auto docsettings = new DdoxSettings;
116 
117 	string[] macrofiles;
118 	string[] overridemacrofiles;
119 	string sitemapurl = "http://127.0.0.1/";
120 	bool lowercasenames;
121 	bool hyphenate;
122 	getopt(args,
123 		//config.passThrough,
124 		"decl-sort", &docsettings.declSort,
125 		"file-name-style", &gensettings.fileNameStyle,
126 		"hyphenate", &hyphenate,
127 		"lowercase-names", &lowercasenames,
128 		"module-sort", &docsettings.moduleSort,
129 		"navigation-type", &gensettings.navigationType,
130 		"override-macros", &overridemacrofiles,
131 		"package-order", &docsettings.packageOrder,
132 		"sitemap-url", &sitemapurl,
133 		"std-macros", &macrofiles,
134 		"enum-member-pages", &gensettings.enumMemberPages,
135 		"html-style", &gensettings.htmlOutputStyle,
136 		);
137 	gensettings.siteUrl = URL(sitemapurl);
138 
139 	if (lowercasenames) gensettings.fileNameStyle = MethodStyle.lowerCase;
140 
141 	if( args.length < 3 ){
142 		showUsage(args);
143 		return 1;
144 	}
145 
146 	setDefaultDdocMacroFiles(macrofiles);
147 	setOverrideDdocMacroFiles(overridemacrofiles);
148 	if (hyphenate) enableHyphenation();
149 
150 	// parse the json output file
151 	pack = parseDocFile(args[2], docsettings);
152 
153 	return 0;
154 }
155 
156 int cmdFilterDocs(string[] args)
157 {
158 	string[] excluded, included;
159 	Protection minprot = Protection.Private;
160 	bool keeputests = false;
161 	bool keepinternals = false;
162 	bool unittestexamples = true;
163 	bool nounittestexamples = false;
164 	bool justdoc = false;
165 	getopt(args,
166 		//config.passThrough,
167 		"ex", &excluded,
168 		"in", &included,
169 		"min-protection", &minprot,
170 		"only-documented", &justdoc,
171 		"keep-unittests", &keeputests,
172 		"keep-internals", &keepinternals,
173 		"unittest-examples", &unittestexamples, // deprecated, kept to not break existing scripts
174 		"no-unittest-examples", &nounittestexamples);
175 
176 	if (keeputests) keepinternals = true;
177 	if (nounittestexamples) unittestexamples = false;
178 
179 	string jsonfile;
180 	if( args.length < 3 ){
181 		showUsage(args);
182 		return 1;
183 	}
184 
185 	Json filterProt(Json json, Json parent, Json last_decl, Json mod)
186 	{
187 		if (last_decl.type == Json.Type.undefined) last_decl = parent;
188 
189 		string templateName(Json j){
190 			auto n = j["name"].opt!string();
191 			auto idx = n.indexOf('(');
192 			if( idx >= 0 ) return n[0 .. idx];
193 			return n;
194 		}
195 
196 		if( json.type == Json.Type.Object ){
197 			auto comment = json["comment"].opt!string;
198 			if( justdoc && comment.empty ){
199 				if( parent.type != Json.Type.Object || parent["kind"].opt!string() != "template" || templateName(parent) != json["name"].opt!string() )
200 					return Json.undefined;
201 			}
202 
203 			Protection prot = Protection.Public;
204 			if( auto p = "protection" in json ){
205 				switch(p.get!string){
206 					default: break;
207 					case "private": prot = Protection.Private; break;
208 					case "package": prot = Protection.Package; break;
209 					case "protected": prot = Protection.Protected; break;
210 				}
211 			}
212 			if( comment.strip == "private" ) prot = Protection.Private;
213 			if( prot < minprot ) return Json.undefined;
214 
215 			auto name = json["name"].opt!string();
216 			bool is_internal = name.startsWith("__");
217 			bool is_unittest = name.startsWith("__unittest");
218 			if (name.startsWith("_staticCtor") || name.startsWith("_staticDtor")) is_internal = true;
219 			else if (name.startsWith("_sharedStaticCtor") || name.startsWith("_sharedStaticDtor")) is_internal = true;
220 
221 			if (unittestexamples && is_unittest && !comment.empty) {
222 				assert(last_decl.type == Json.Type.object, "Don't have a last_decl context.");
223 				try {
224 					string source = extractUnittestSourceCode(json, mod);
225 					if (last_decl["comment"].opt!string.empty) {
226 						writefln("Warning: Cannot add documented unit test %s to %s, which is not documented.", name, last_decl["name"].opt!string);
227 					} else {
228 						last_decl["comment"] ~= format("Example:\n%s$(DDOX_UNITTEST_HEADER %s)\n---\n%s\n---\n$(DDOX_UNITTEST_FOOTER %s)\n", comment.strip, name, source, name);
229 					}
230 				} catch (Exception e) {
231 					writefln("Failed to add documented unit test %s:%s as example: %s",
232 						mod["file"].get!string(), json["line"].get!long, e.msg);
233 					return Json.undefined;
234 				}
235 			}
236 
237 			if (!keepinternals && is_internal) return Json.undefined;
238 
239 			if (!keeputests && is_unittest) return Json.undefined;
240 
241 			if (auto mem = "members" in json)
242 				json["members"] = filterProt(*mem, json, Json.undefined, mod);
243 		} else if( json.type == Json.Type.Array ){
244 			auto last_child_decl = Json.undefined;
245 			Json[] newmem;
246 			foreach (m; json) {
247 				auto mf = filterProt(m, parent, last_child_decl, mod);
248 				if (mf.type == Json.Type.undefined) continue;
249 				if (mf.type == Json.Type.object && !mf["name"].opt!string.startsWith("__unittest") && icmp(mf["comment"].opt!string.strip, "ditto") != 0)
250 					last_child_decl = mf;
251 				newmem ~= mf;
252 			}
253 			return Json(newmem);
254 		}
255 		return json;
256 	}
257 
258 	writefln("Reading doc file...");
259 	auto text = readText(args[2]);
260 	int line = 1;
261 	writefln("Parsing JSON...");
262 	auto json = parseJson(text, &line);
263 
264 	writefln("Filtering modules...");
265 	Json[] dst;
266 	foreach (m; json) {
267 		if ("name" !in m) {
268 			writefln("No name for module %s - ignoring", m["file"].opt!string);
269 			continue;
270 		}
271 		auto n = m["name"].get!string;
272 		bool include = true;
273 		foreach (ex; excluded)
274 			if (n.startsWith(ex)) {
275 				include = false;
276 				break;
277 			}
278 		foreach (inc; included)
279 			if (n.startsWith(inc)) {
280 				include = true;
281 				break;
282 			}
283 		if (include) {
284 			auto doc = filterProt(m, Json.undefined, Json.undefined, m);
285 			if (doc.type != Json.Type.undefined)
286 				dst ~= doc;
287                 }
288 	}
289 
290 	writefln("Writing filtered docs...");
291 	auto buf = appender!string();
292 	writePrettyJsonString(buf, Json(dst));
293 	std.file.write(args[2], buf.data());
294 
295 	return 0;
296 }
297 
298 Package parseDocFile(string filename, DdoxSettings settings)
299 {
300 	writefln("Reading doc file...");
301 	auto text = readText(filename);
302 	int line = 1;
303 	writefln("Parsing JSON...");
304 	auto json = parseJson(text, &line);
305 	writefln("Parsing docs...");
306 	Package root;
307 	root = parseJsonDocs(json);
308 	writefln("Finished parsing docs.");
309 
310 	processDocs(root, settings);
311 	return root;
312 }
313 
314 void showUsage(string[] args)
315 {
316 	string cmd;
317 	if( args.length >= 2 ) cmd = args[1];
318 
319 	switch(cmd){
320 		default:
321 			writefln(
322 `Usage: %s <COMMAND> [args...]
323 
324     <COMMAND> can be one of:
325         generate-html
326         serve-html
327         filter
328 
329  -h --help                 Show this help
330 
331 Use <COMMAND> -h|--help to get detailed usage information for a command.
332 `, args[0]);
333 			break;
334 		case "serve-html":
335 			writefln(
336 `Usage: %s serve-html <ddocx-input-file>
337     --std-macros=FILE      File containing DDOC macros that will be available
338     --override-macros=FILE File containing DDOC macros that will override local
339                            definitions (Macros: section)
340     --navigation-type=TYPE Change the type of navigation (ModuleList,
341                            ModuleTree (default), DeclarationTree)
342     --package-order=NAME   Causes the specified module to be ordered first. Can
343                            be specified multiple times.
344     --sitemap-url          Specifies the base URL used for sitemap generation
345     --module-sort=MODE     The sort order used for lists of modules
346     --decl-sort=MODE       The sort order used for declaration lists
347     --web-file-dir=DIR     Make files from dir available on the served site
348     --enum-member-pages    Generate a single page per enum member
349     --html-style=STYLE     Sets the HTML output style, either pretty (default)
350                            or compact.
351     --hyphenate            hyphenate text
352  -h --help                 Show this help
353 
354 The following values can be used as sorting modes: none, name, protectionName,
355 protectionInheritanceName
356 `, args[0]);
357 			break;
358 		case "generate-html":
359 			writefln(
360 `Usage: %s generate-html <ddocx-input-file> <output-dir>
361     --std-macros=FILE      File containing DDOC macros that will be available
362     --override-macros=FILE File containing DDOC macros that will override local
363                            definitions (Macros: section)
364     --navigation-type=TYPE Change the type of navigation (ModuleList,
365                            ModuleTree, DeclarationTree)
366     --package-order=NAME   Causes the specified module to be ordered first. Can
367                            be specified multiple times.
368     --sitemap-url          Specifies the base URL used for sitemap generation
369     --module-sort=MODE     The sort order used for lists of modules
370     --decl-sort=MODE       The sort order used for declaration lists
371     --file-name-style=STY  Sets a translation style for symbol names to file
372                            names. Use this instead of --lowercase-name.
373                            Possible values for STY:
374                              unaltered, camelCase, pascalCase, lowerCase,
375                              upperCase, lowerUnderscored, upperUnderscored
376     --lowercase-names      DEPRECATED: Outputs all file names in lower case.
377                            This option is useful on case insensitive file
378                            systems.
379     --enum-member-pages    Generate a single page per enum member
380     --html-style=STYLE     Sets the HTML output style, either pretty (default)
381                            compact or .
382     --hyphenate            hyphenate text
383  -h --help                 Show this help
384 
385 The following values can be used as sorting modes: none, name, protectionName,
386 protectionInheritanceName
387 `, args[0]);
388 			break;
389 		case "filter":
390 			writefln(
391 `Usage: %s filter <ddocx-input-file> [options]
392     --ex=PREFIX            Exclude modules with prefix
393     --in=PREFIX            Force include of modules with prefix
394     --min-protection=PROT  Remove items with lower protection level than
395                            specified.
396                            PROT can be: Public, Protected, Package, Private
397     --only-documented      Remove undocumented entities.
398     --keep-unittests       Do not remove unit tests from documentation.
399                            Implies --keep-internals.
400     --keep-internals       Do not remove symbols starting with two underscores.
401     --unittest-examples    Add documented unit tests as examples to the
402                            preceding declaration (deprecated, enabled by
403                            default)
404     --no-unittest-examples Don't convert documented unit tests to examples
405  -h --help                 Show this help
406 `, args[0]);
407 	}
408 	if( args.length < 2 ){
409 	} else {
410 
411 	}
412 }
413 
414 private string extractUnittestSourceCode(Json decl, Json mod)
415 {
416 	auto filename = mod["file"].get!string();
417 	enforce("line" in decl && "endline" in decl, "Missing line/endline fields.");
418 	auto from = decl["line"].get!long;
419 	auto to = decl["endline"].get!long;
420 
421 	// read the matching lines out of the file
422 	auto app = appender!string();
423 	long lc = 1;
424 	foreach (str; File(filename).byLine) {
425 		if (lc >= from) {
426 			app.put(str);
427 			app.put('\n');
428 		}
429 		if (++lc > to) break;
430 	}
431 	auto ret = app.data;
432 
433 	// strip the "unittest { .. }" surroundings
434 	auto idx = ret.indexOf("unittest");
435 	enforce(idx >= 0, format("Missing 'unittest' for unit test at %s:%s.", filename, from));
436 	ret = ret[idx .. $];
437 
438 	idx = ret.indexOf("{");
439 	enforce(idx >= 0, format("Missing opening '{' for unit test at %s:%s.", filename, from));
440 	ret = ret[idx+1 .. $];
441 
442 	idx = ret.lastIndexOf("}");
443 	enforce(idx >= 0, format("Missing closing '}' for unit test at %s:%s.", filename, from));
444 	ret = ret[0 .. idx];
445 
446 	// unindent lines according to the indentation of the first line
447 	app = appender!string();
448 	string indent;
449 	foreach (i, ln; ret.splitLines) {
450 		if (i == 1) {
451 			foreach (j; 0 .. ln.length)
452 				if (ln[j] != ' ' && ln[j] != '\t') {
453 					indent = ln[0 .. j];
454 					break;
455 				}
456 		}
457 		if (i > 0 || ln.strip.length > 0) {
458 			size_t j = 0;
459 			while (j < indent.length && !ln.empty) {
460 				if (ln.front != indent[j]) break;
461 				ln.popFront();
462 				j++;
463 			}
464 			app.put(ln);
465 			app.put('\n');
466 		}
467 	}
468 	return app.data;
469 }