1 /**
2 	DietDoc/DDOC support routines
3 
4 	Copyright: © 2012-2015 RejectedSoftware e.K.
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig
7 */
8 module ddox.ddoc;
9 
10 import vibe.core.log;
11 import vibe.utils.string;
12 
13 import hyphenate : Hyphenator;
14 
15 import std.algorithm : canFind, countUntil, map, min, remove;
16 import std.array;
17 import std.conv;
18 import std.string;
19 import std.uni : isAlpha;
20 
21 // TODO: support escapes section
22 
23 
24 shared static this()
25 {
26 	s_standardMacros =
27 		[
28 		 `B`: `<b>$0</b>`,
29 		 `I`: `<i>$0</i>`,
30 		 `U`: `<u>$0</u>`,
31 		 `P` : `<p>$0</p>`,
32 		 `DL` : `<dl>$0</dl>`,
33 		 `DT` : `<dt>$0</dt>`,
34 		 `DD` : `<dd>$0</dd>`,
35 		 `TABLE` : `<table>$0</table>`,
36 		 `TR` : `<tr>$0</tr>`,
37 		 `TH` : `<th>$0</th>`,
38 		 `TD` : `<td>$0</td>`,
39 		 `OL` : `<ol>$0</ol>`,
40 		 `UL` : `<ul>$0</ul>`,
41 		 `LI` : `<li>$0</li>`,
42 		 `LINK` : `<a href="$0">$0</a>`,
43 		 `LINK2` : `<a href="$1">$+</a>`,
44 		 `LPAREN` : `(`,
45 		 `RPAREN` : `)`,
46 
47 		 `RED` :   `<font color=red>$0</font>`,
48 		 `BLUE` :  `<font color=blue>$0</font>`,
49 		 `GREEN` : `<font color=green>$0</font>`,
50 		 `YELLOW` : `<font color=yellow>$0</font>`,
51 		 `BLACK` : `<font color=black>$0</font>`,
52 		 `WHITE` : `<font color=white>$0</font>`,
53 
54 		 `D_CODE` : `<pre class="d_code">$0</pre>`,
55 		 `D_COMMENT` : `$(GREEN $0)`,
56 		 `D_STRING`  : `$(RED $0)`,
57 		 `D_KEYWORD` : `$(BLUE $0)`,
58 		 `D_PSYMBOL` : `$(U $0)`,
59 		 `D_PARAM` : `$(I $0)`,
60 		 `BACKTICK`: "`",
61 		 `DDOC_BACKQUOTED`: `$(D_INLINECODE $0)`,
62 		 //`D_INLINECODE`: `<pre style="display:inline;" class="d_inline_code">$0</pre>`,
63 		 `D_INLINECODE`: `<code class="lang-d">$0</code>`,
64 
65 		 `DDOC` : `<html>
66   <head>
67     <META http-equiv="content-type" content="text/html; charset=utf-8">
68     <title>$(TITLE)</title>
69   </head>
70   <body>
71   <h1>$(TITLE)</h1>
72   $(BODY)
73   </body>
74 </html>`,
75 
76 		 `DDOC_COMMENT` : `<!-- $0 -->`,
77 		 `DDOC_DECL` : `$(DT $(BIG $0))`,
78 		 `DDOC_DECL_DD` : `$(DD $0)`,
79 		 `DDOC_DITTO` : `$(BR)$0`,
80 		 `DDOC_SECTIONS` : `$0`,
81 		 `DDOC_SUMMARY` : `$0$(BR)$(BR)`,
82 		 `DDOC_DESCRIPTION` : `$0$(BR)$(BR)`,
83 		 `DDOC_AUTHORS` : "$(B Authors:)$(BR)\n$0$(BR)$(BR)",
84 		 `DDOC_BUGS` : "$(RED BUGS:)$(BR)\n$0$(BR)$(BR)",
85 		 `DDOC_COPYRIGHT` : "$(B Copyright:)$(BR)\n$0$(BR)$(BR)",
86 		 `DDOC_DATE` : "$(B Date:)$(BR)\n$0$(BR)$(BR)",
87 		 `DDOC_DEPRECATED` : "$(RED Deprecated:)$(BR)\n$0$(BR)$(BR)",
88 		 `DDOC_EXAMPLES` : "$(B Examples:)$(BR)\n$0$(BR)$(BR)",
89 		 `DDOC_HISTORY` : "$(B History:)$(BR)\n$0$(BR)$(BR)",
90 		 `DDOC_LICENSE` : "$(B License:)$(BR)\n$0$(BR)$(BR)",
91 		 `DDOC_RETURNS` : "$(B Returns:)$(BR)\n$0$(BR)$(BR)",
92 		 `DDOC_SEE_ALSO` : "$(B See Also:)$(BR)\n$0$(BR)$(BR)",
93 		 `DDOC_STANDARDS` : "$(B Standards:)$(BR)\n$0$(BR)$(BR)",
94 		 `DDOC_THROWS` : "$(B Throws:)$(BR)\n$0$(BR)$(BR)",
95 		 `DDOC_VERSION` : "$(B Version:)$(BR)\n$0$(BR)$(BR)",
96 		 `DDOC_SECTION_H` : `$(B $0)$(BR)$(BR)`,
97 		 `DDOC_SECTION` : `$0$(BR)$(BR)`,
98 		 `DDOC_MEMBERS` : `$(DL $0)`,
99 		 `DDOC_MODULE_MEMBERS` : `$(DDOC_MEMBERS $0)`,
100 		 `DDOC_CLASS_MEMBERS` : `$(DDOC_MEMBERS $0)`,
101 		 `DDOC_STRUCT_MEMBERS` : `$(DDOC_MEMBERS $0)`,
102 		 `DDOC_ENUM_MEMBERS` : `$(DDOC_MEMBERS $0)`,
103 		 `DDOC_TEMPLATE_MEMBERS` : `$(DDOC_MEMBERS $0)`,
104 		 `DDOC_PARAMS` : "$(B Params:)$(BR)\n$(TABLE $0)$(BR)",
105 		 `DDOC_PARAM_ROW` : `$(TR $0)`,
106 		 `DDOC_PARAM_ID` : `$(TD $0)`,
107 		 `DDOC_PARAM_DESC` : `$(TD $0)`,
108 		 `DDOC_BLANKLINE` : `$(BR)$(BR)`,
109 
110 		 `DDOC_ANCHOR` : `<a name="$1"></a>`,
111 		 `DDOC_PSYMBOL` : `$(U $0)`,
112 		 `DDOC_KEYWORD` : `$(B $0)`,
113 		 `DDOC_PARAM` : `$(I $0)`,
114 		 ];
115 	import std.datetime : Clock;
116 	auto now = Clock.currTime();
117 	s_standardMacros["DATETIME"] = "%s %s %s %s:%s:%s %s".format(
118 		now.dayOfWeek.to!string.capitalize, now.month.to!string.capitalize,
119 		now.day, now.hour, now.minute, now.second, now.year);
120 	s_standardMacros["YEAR"] = now.year.to!string;
121 }
122 
123 
124 /**
125 	Takes a DDOC string and outputs formatted HTML.
126 
127 	The hlevel parameter specifies the header level used for section names (&lt;h2&gt by default).
128 	By specifying a display_section callback it is also possible to output only certain sections.
129 */
130 string formatDdocComment(string ddoc_, int hlevel = 2, bool delegate(string) display_section = null)
131 {
132 	return formatDdocComment(ddoc_, new BareContext, hlevel, display_section);
133 }
134 /// ditto
135 string formatDdocComment(string text, DdocContext context, int hlevel = 2, bool delegate(string) display_section = null)
136 {
137 	auto dst = appender!string();
138 	filterDdocComment(dst, text, context, hlevel, display_section);
139 	return dst.data;
140 }
141 /// ditto
142 void filterDdocComment(R)(ref R dst, string text, DdocContext context, int hlevel = 2, bool delegate(string) display_section = null)
143 {
144 	auto comment = new DdocComment(text);
145 	comment.renderSectionsR(dst, context, display_section, hlevel);
146 }
147 
148 
149 /**
150 	Sets a set of macros that will be available to all calls to formatDdocComment.
151 */
152 void setDefaultDdocMacroFiles(string[] filenames)
153 {
154 	import vibe.core.file;
155 	import vibe.stream.operations;
156 	s_defaultMacros = null;
157 	foreach (filename; filenames) {
158 		auto text = readAllUTF8(openFile(filename));
159 		parseMacros(s_defaultMacros, splitLines(text));
160 	}
161 }
162 
163 
164 /**
165 	Sets a set of macros that will be available to all calls to formatDdocComment and override local macro definitions.
166 */
167 void setOverrideDdocMacroFiles(string[] filenames)
168 {
169 	import vibe.core.file;
170 	import vibe.stream.operations;
171 	s_overrideMacros = null;
172 	foreach (filename; filenames) {
173 		auto text = readAllUTF8(openFile(filename));
174 		parseMacros(s_overrideMacros, splitLines(text));
175 	}
176 }
177 
178 
179 /**
180    Enable hyphenation of doc text.
181 */
182 void enableHyphenation()
183 {
184 	s_hyphenator = Hyphenator(import("hyphen.tex")); // en-US
185 	s_enableHyphenation = true;
186 }
187 
188 
189 void hyphenate(R)(in char[] word, R orng)
190 {
191 	s_hyphenator.hyphenate(word, "\&shy;", s => orng.put(s));
192 }
193 
194 /**
195 	Holds a DDOC comment and formats it sectionwise as HTML.
196 */
197 class DdocComment {
198 	private {
199 		Section[] m_sections;
200 		string[string] m_macros;
201 		bool m_isDitto = false;
202 		bool m_isPrivate = false;
203 	}
204 
205 	this(string text)
206 	{
207 
208 		if (text.strip.icmp("ditto") == 0) { m_isDitto = true; return; }
209 		if (text.strip.icmp("private") == 0) { m_isPrivate = true; return; }
210 
211 		auto lines = splitLines(text);
212 		if( !lines.length ) return;
213 
214 		int getLineType(int i)
215 		{
216 			auto ln = strip(lines[i]);
217 			if( ln.length == 0 ) return BLANK;
218 			else if( ln.length >= 3 && ln.allOf("-") ) return CODE;
219 			else if( ln.indexOf(':') > 0 && isIdent(ln[0 .. ln.indexOf(':')]) ) return SECTION;
220 			return TEXT;
221 		}
222 
223 		int skipCodeBlock(int start)
224 		{
225 			do {
226 				start++;
227 			} while(start < lines.length && getLineType(start) != CODE);
228 			if (start >= lines.length) return start; // unterminated code section
229 			return start+1;
230 		}
231 
232 		int skipSection(int start)
233 		{
234 			while (start < lines.length) {
235 				if (getLineType(start) == SECTION) break;
236 				if (getLineType(start) == CODE)
237 					start = skipCodeBlock(start);
238 				else start++;
239 			}
240 			return start;
241 		}
242 
243 		int skipBlock(int start)
244 		{
245 			do {
246 				start++;
247 			} while(start < lines.length && getLineType(start) == TEXT);
248 			return start;
249 		}
250 
251 
252 		int i = 0;
253 
254 		// special case short description on the first line
255 		while( i < lines.length && getLineType(i) == BLANK ) i++;
256 		if( i < lines.length && getLineType(i) == TEXT ){
257 			auto j = skipBlock(i);
258 			m_sections ~= Section("$Short", lines[i .. j]);
259 			i = j;
260 		}
261 
262 		// first section is implicitly the long description
263 		{
264 			auto j = skipSection(i);
265 			if( j > i ){
266 				m_sections ~= Section("$Long", lines[i .. j]);
267 				i = j;
268 			}
269 		}
270 
271 		// parse all other sections
272 		while( i < lines.length ){
273 			assert(getLineType(i) == SECTION);
274 			auto j = skipSection(i+1);
275 			assert(j <= lines.length);
276 			auto pidx = lines[i].indexOf(':');
277 			auto sect = strip(lines[i][0 .. pidx]);
278 			lines[i] = stripLeftDD(lines[i][pidx+1 .. $]);
279 			if (lines[i].empty && i < lines.length) i++;
280 			if (sect == "Macros") parseMacros(m_macros, lines[i .. j]);
281 			else {
282 				m_sections ~= Section(sect, lines[i .. j]);
283 			}
284 			i = j;
285 		}
286 	}
287 
288 	@property bool isDitto() const { return m_isDitto; }
289 	@property bool isPrivate() const { return m_isPrivate; }
290 
291 	/// The macros contained in the "Macros" section (if any)
292 	@property const(string[string]) macros() const { return m_macros; }
293 
294 	bool hasSection(string name) const { return m_sections.canFind!(s => s.name == name); }
295 
296 	void renderSectionR(R)(ref R dst, DdocContext context, string name, int hlevel = 2)
297 	{
298 		renderSectionsR(dst, context, s => s == name, hlevel);
299 	}
300 
301 	void renderSectionsR(R)(ref R dst, DdocContext context, scope bool delegate(string) display_section, int hlevel)
302 	{
303 		string[string] allmacros;
304 		foreach (k, v; context.defaultMacroDefinitions) allmacros[k] = v;
305 		foreach (k, v; m_macros) allmacros[k] = v;
306 		foreach (k, v; context.overrideMacroDefinitions) allmacros[k] = v;
307 
308 		foreach (s; m_sections) {
309 			if (display_section && !display_section(s.name)) continue;
310 			parseSection(dst, s.name, s.lines, context, hlevel, allmacros);
311 		}
312 	}
313 
314 	string renderSection(DdocContext context, string name, int hlevel = 2)
315 	{
316 		auto dst = appender!string();
317 		renderSectionR(dst, context, name, hlevel);
318 		return dst.data;
319 	}
320 
321 	string renderSections(DdocContext context, bool delegate(string) display_section, int hlevel)
322 	{
323 		auto dst = appender!string();
324 		renderSectionsR(dst, context, display_section, hlevel);
325 		return dst.data;
326 	}
327 }
328 
329 
330 /**
331 	Provides context information about the documented element.
332 */
333 interface DdocContext {
334 	/// A line array with macro definitions
335 	@property string[string] defaultMacroDefinitions();
336 
337 	/// Line array with macro definitions that take precedence over local macros
338 	@property string[string] overrideMacroDefinitions();
339 
340 	/// Looks up a symbol in the scope of the documented element and returns a link to it.
341 	string lookupScopeSymbolLink(string name);
342 }
343 
344 
345 private class BareContext : DdocContext {
346 	@property string[string] defaultMacroDefinitions() { return null; }
347 	@property string[string] overrideMacroDefinitions() { return null; }
348 	string lookupScopeSymbolLink(string name) { return null; }
349 }
350 
351 private enum {
352 	BLANK,
353 	TEXT,
354 	CODE,
355 	SECTION
356 }
357 
358 private struct Section {
359 	string name;
360 	string[] lines;
361 
362 	this(string name, string[] lines...)
363 	{
364 		this.name = name;
365 		this.lines = lines;
366 	}
367 }
368 
369 private {
370 	immutable string[string] s_standardMacros;
371 	string[string] s_defaultMacros;
372 	string[string] s_overrideMacros;
373 	bool s_enableHyphenation;
374 	Hyphenator s_hyphenator;
375 }
376 
377 /// private
378 private void parseSection(R)(ref R dst, string sect, string[] lines, DdocContext context, int hlevel, string[string] macros)
379 {
380 	if( sect == "$Short" ) hlevel = -1;
381 
382 	void putHeader(string hdr){
383 		if( hlevel <= 0 ) return;
384 		dst.put("<section>");
385 		if( sect.length > 0 && sect[0] != '$' ){
386 			dst.put("<h"~to!string(hlevel)~">");
387 			foreach( ch; hdr ) dst.put(ch == '_' ? ' ' : ch);
388 			dst.put("</h"~to!string(hlevel)~">\n");
389 		}
390 	}
391 
392 	void putFooter(){
393 		if( hlevel <= 0 ) return;
394 		dst.put("</section>\n");
395 	}
396 
397 	int getLineType(int i)
398 	{
399 		auto ln = strip(lines[i]);
400 		if( ln.length == 0 ) return BLANK;
401 		else if( ln.length >= 3 &&ln.allOf("-") ) return CODE;
402 		else if( ln.indexOf(':') > 0 && !ln[0 .. ln.indexOf(':')].anyOf(" \t") ) return SECTION;
403 		return TEXT;
404 	}
405 
406 	int skipBlock(int start)
407 	{
408 		do {
409 			start++;
410 		} while(start < lines.length && getLineType(start) == TEXT);
411 		return start;
412 	}
413 
414 	int skipCodeBlock(int start)
415 	{
416 		do {
417 			start++;
418 		} while(start < lines.length && getLineType(start) != CODE);
419 		return start;
420 	}
421 
422 	// handle backtick inline-code
423 	for (int i = 0; i < lines.length; i++) {
424 		int lntype = getLineType(i);
425 		if (lntype == CODE) i = skipCodeBlock(i);
426 		else if (sect == "Params") {
427 			auto idx = lines[i].indexOf('=');
428 			if (idx > 0 && isIdent(lines[i][0 .. idx].strip)) {
429 				lines[i] = lines[i][0 .. idx+1] ~ lines[i][idx+1 .. $].highlightAndCrossLink(context);
430 			} else {
431 				lines[i] = lines[i].highlightAndCrossLink(context);
432 			}
433 		} else lines[i] = lines[i].highlightAndCrossLink(context);
434 	}
435 	lines = renderMacros(lines.join("\n").stripDD, context, macros).splitLines();
436 
437 	switch( sect ){
438 		default:
439 			putHeader(sect);
440 			int i = 0;
441 			while( i < lines.length ){
442 				int lntype = getLineType(i);
443 
444 				switch( lntype ){
445 					default: assert(false, "Unexpected line type "~to!string(lntype)~": "~lines[i]);
446 					case BLANK:
447 						dst.put('\n');
448 						i++;
449 						continue;
450 					case SECTION:
451 					case TEXT:
452 						if( hlevel >= 0 ) dst.put("<p>");
453 						auto j = skipBlock(i);
454 						bool first = true;
455 						renderTextLine(dst, lines[i .. j].join("\n")/*.stripDD*/, context);
456 						dst.put('\n');
457 						if( hlevel >= 0 ) dst.put("</p>\n");
458 						i = j;
459 						break;
460 					case CODE:
461 						dst.put("<pre class=\"code\"><code class=\"lang-d\">");
462 						auto j = skipCodeBlock(i);
463 						auto base_indent = baseIndent(lines[i+1 .. j]);
464 						renderCodeLine(dst, lines[i+1 .. j].map!(ln => ln.unindent(base_indent)).join("\n"), context);
465 						dst.put("</code></pre>\n");
466 						i = j+1;
467 						break;
468 				}
469 			}
470 			putFooter();
471 			break;
472 		case "Params":
473 			putHeader("Parameters");
474 			dst.put("<table><col class=\"caption\"><tr><th>Name</th><th>Description</th></tr>\n");
475 			bool in_parameter = false;
476 			string desc;
477 			foreach( string ln; lines ){
478 				// check if the line starts a parameter documentation
479 				string name;
480 				auto eidx = ln.indexOf("=");
481 				if( eidx > 0 ) name = ln[0 .. eidx].strip();
482 				if( !isIdent(name) ) name = null;
483 
484 				// if it does, start a new row
485 				if( name.length ){
486 					if( in_parameter ){
487 						renderTextLine(dst, desc, context);
488 						dst.put("</td></tr>\n");
489 					}
490 
491 					dst.put("<tr><td id=\"");
492 					dst.put(name);
493 					dst.put("\">");
494 					dst.put(name);
495 					dst.put("</td><td>");
496 
497 					desc = ln[eidx+1 .. $];
498 					in_parameter = true;
499 				} else if( in_parameter ) desc ~= "\n" ~ ln;
500 			}
501 
502 			if( in_parameter ){
503 				renderTextLine(dst, desc, context);
504 				dst.put("</td></tr>\n");
505 			}
506 
507 			dst.put("</table>\n");
508 			putFooter();
509 			break;
510 	}
511 }
512 
513 private string highlightAndCrossLink(string line, DdocContext context)
514 {
515 	auto dst = appender!string;
516 	highlightAndCrossLink(dst, line, context);
517 	return dst.data;
518 }
519 
520 private void highlightAndCrossLink(R)(ref R dst, string line, DdocContext context)
521 {
522 	size_t inCode;
523 	while( line.length > 0 ){
524 		switch( line[0] ){
525 			default:
526 				dst.put(line[0]);
527 				line = line[1 .. $];
528 				break;
529 			case '<':
530 				auto res = skipHtmlTag(line);
531 				if (res.startsWith("<code"))
532 					++inCode;
533 				else if (res == "</code>")
534 					--inCode;
535 				dst.put(res);
536 				break;
537 			case '_':
538 				line = line[1 .. $];
539 				auto ident = skipIdent(line);
540 				if( ident.length )
541 				{
542 					if (s_enableHyphenation && !inCode)
543 						hyphenate(ident, dst);
544 					else
545 						dst.put(ident);
546 				}
547 				else dst.put('_');
548 				break;
549 			case '`':
550 				line.popFront();
551 				auto idx = line.indexOf('`');
552 				if (idx < 0) break;
553 				dst.put("$(DDOC_BACKQUOTED ");
554 				foreach (i; 0 .. idx) {
555 					switch (line[i]) {
556 						default: dst.put(line[i]); break;
557 						case '&': dst.put("&amp;"); break;
558 						case '<': dst.put("&lt;"); break;
559 						case '>': dst.put("&gt;"); break;
560 						case '(': dst.put("$(LPAREN)"); break;
561 						case ')': dst.put("$(RPAREN)"); break;
562 					}
563 				}
564 				dst.put(')');
565 				line = line[idx .. $];
566 				break;
567 			case '.':
568 				if (line.length > 1 && (line[1 .. $].front.isAlpha || line[1] == '_')) goto case;
569 				else goto default;
570 			case 'a': .. case 'z':
571 			case 'A': .. case 'Z':
572 
573 				auto url = skipUrl(line);
574 				if( url.length ){
575 					/*dst.put("<a href=\"");
576 					dst.put(url);
577 					dst.put("\">");*/
578 					dst.put(url);
579 					//dst.put("</a>");
580 					break;
581 				}
582 
583 				auto ident = skipIdent(line);
584 				auto link = context.lookupScopeSymbolLink(ident);
585 				if (link.length) {
586 					import ddox.highlight : highlightDCode;
587 					if( link != "#" ){
588 						dst.put("<a href=\"");
589 						dst.put(link);
590 						dst.put("\">");
591 					}
592 					if (!inCode) dst.put("<code class=\"lang-d\">");
593 					dst.highlightDCode(ident, null);
594 					if (!inCode) dst.put("</code>");
595 					if( link != "#" ) dst.put("</a>");
596 				} else {
597 					ident = ident.replace("._", ".");
598 					if (s_enableHyphenation && !inCode)
599 						hyphenate(ident, dst);
600 					else
601 						dst.put(ident);
602 				}
603 				break;
604 		}
605 	}
606 }
607 
608 /// private
609 private void renderTextLine(R)(ref R dst, string line, DdocContext context)
610 {
611 	while( line.length > 0 ){
612 		switch( line[0] ){
613 			default:
614 				dst.put(line[0]);
615 				line = line[1 .. $];
616 				break;
617 			case '<':
618 				dst.put(skipHtmlTag(line));
619 				break;
620 			case '>':
621 				dst.put("&gt;");
622 				line.popFront();
623 				break;
624 			case '&':
625 				if (line.length >= 2 && (line[1].isAlpha || line[1] == '#')) dst.put('&');
626 				else dst.put("&amp;");
627 				line.popFront();
628 				break;
629 		}
630 	}
631 }
632 
633 /// private
634 private void renderCodeLine(R)(ref R dst, string line, DdocContext context)
635 {
636 	import ddox.highlight : highlightDCode;
637 	dst.highlightDCode(line, (string ident, scope void delegate() insert_ident) {
638 		auto link = context.lookupScopeSymbolLink(ident);
639 		if (link.length && link != "#") {
640 			dst.put("<a href=\"");
641 			dst.put(link);
642 			dst.put("\">");
643 			insert_ident();
644 			dst.put("</a>");
645 		} else insert_ident();
646 	});
647 }
648 
649 /// private
650 private void renderMacros(R)(ref R dst, string line, DdocContext context, string[string] macros, string[] params = null, MacroInvocation[] callstack = null)
651 {
652 	while( !line.empty ){
653 		auto idx = line.indexOf('$');
654 		if( idx < 0 ){
655 			dst.put(line);
656 			return;
657 		}
658 		dst.put(line[0 .. idx]);
659 		line = line[idx .. $];
660 		renderMacro(dst, line, context, macros, params, callstack);
661 	}
662 }
663 
664 /// private
665 private string renderMacros(string line, DdocContext context, string[string] macros, string[] params = null, MacroInvocation[] callstack = null)
666 {
667 	auto app = appender!string;
668 	renderMacros(app, line, context, macros, params, callstack);
669 	return app.data;
670 }
671 
672 /// private
673 private void renderMacro(R)(ref R dst, ref string line, DdocContext context, string[string] macros, string[] params, MacroInvocation[] callstack)
674 {
675 	assert(line[0] == '$');
676 	line = line[1 .. $];
677 	if( line.length < 1) {
678 		dst.put("$");
679 		return;
680 	}
681 
682 	if( line[0] >= '0' && line[0] <= '9' ){
683 		int pidx = line[0]-'0';
684 		if( pidx < params.length )
685 			dst.put(params[pidx]);
686 		line = line[1 .. $];
687 	} else if( line[0] == '+' ){
688 		if( params.length ){
689 			auto idx = params[0].indexOf(',');
690 			if( idx >= 0 ) dst.put(params[0][idx+1 .. $].specialStrip());
691 		}
692 		line = line[1 .. $];
693 	} else if( line[0] == '(' ){
694 		line = line[1 .. $];
695 		int l = 1;
696 		size_t cidx = 0;
697 		for( cidx = 0; cidx < line.length && l > 0; cidx++ ){
698 			if( line[cidx] == '(' ) l++;
699 			else if( line[cidx] == ')' ) l--;
700 		}
701 		if( l > 0 ){
702 			logDebug("Unmatched parenthesis in DDOC comment: %s", line[0 .. cidx]);
703 			dst.put("(");
704 			return;
705 		}
706 		if( cidx < 1 ){
707 			logDebug("Empty macro parens.");
708 			return;
709 		}
710 
711 		auto mnameidx = line[0 .. cidx-1].countUntilAny(", \t\r\n");
712 		if( mnameidx < 0 ) mnameidx = cidx-1;
713 		if( mnameidx == 0 ){
714 			logDebug("Macro call in DDOC comment is missing macro name.");
715 			return;
716 		}
717 
718 		auto mname = line[0 .. mnameidx];
719 		string rawargtext = line[mnameidx .. cidx-1];
720 
721 		string[] args;
722 		if (rawargtext.length) {
723 			auto rawargs = splitParams(rawargtext);
724 			foreach( arg; rawargs ){
725 				auto argtext = appender!string();
726 				renderMacros(argtext, arg, context, macros, params, callstack);
727 				auto newargs = splitParams(argtext.data);
728 				if (newargs.length == 0) args ~= ""; // always add at least one argument per raw argument
729 				else args ~= newargs;
730 			}
731 		}
732 		if (args.length == 1 && args[0].specialStrip.length == 0) args = null; // remove a single empty argument
733 
734 		args = join(args, ",").specialStrip() ~ args.map!(a => a.specialStrip).array;
735 
736 		logTrace("PARAMS for %s: %s", mname, args);
737 		line = line[cidx .. $];
738 
739 		// check for recursion termination conditions
740 		foreach_reverse (ref c; callstack) {
741 			if (c.name == mname && (args.length <= 1 || args == c.params)) {
742 				logTrace("Terminating recursive macro call of %s: %s", mname, params.length <= 1 ? "no argument text" : "same arguments as previous invocation");
743 				//line = line[cidx .. $];
744 				return;
745 			}
746 		}
747 		callstack.assumeSafeAppend();
748 		callstack ~= MacroInvocation(mname, args);
749 
750 
751 		const(string)* pm = mname in s_overrideMacros;
752 		if( !pm ) pm = mname in macros;
753 		if( !pm ) pm = mname in s_defaultMacros;
754 		if( !pm ) pm = mname in s_standardMacros;
755 
756 		if( pm ){
757 			logTrace("MACRO %s: %s", mname, *pm);
758 			renderMacros(dst, *pm, context, macros, args, callstack);
759 		} else {
760 			logTrace("Macro '%s' not found.", mname);
761 			if( args.length ) dst.put(args[0]);
762 		}
763 	} else dst.put("$");
764 }
765 
766 private struct MacroInvocation {
767 	string name;
768 	string[] params;
769 }
770 
771 private string[] splitParams(string ln)
772 {
773 	string[] ret;
774 	size_t i = 0, start = 0;
775 	while(i < ln.length){
776 		if( ln[i] == ',' ){
777 			ret ~= ln[start .. i];
778 			start = ++i;
779 		} else if( ln[i] == '(' ){
780 			i++;
781 			int l = 1;
782 			for( ; i < ln.length && l > 0; i++ ){
783 				if( ln[i] == '(' ) l++;
784 				else if( ln[i] == ')' ) l--;
785 			}
786 		} else i++;
787 	}
788 	if( i > start ) ret ~= ln[start .. i];
789 	return ret;
790 }
791 
792 private string skipHtmlTag(ref string ln)
793 {
794 	assert(ln[0] == '<');
795 
796 	// skip HTML comment
797 	if (ln.startsWith("<!--")) {
798 		auto idx = ln[4 .. $].indexOf("-->");
799 		if (idx < 0) {
800 			ln.popFront();
801 			return "&lt;";
802 		}
803 		auto ret = ln[0 .. idx+7];
804 		ln = ln[ret.length .. $];
805 		return ret;
806 	}
807 
808 	// too short for a tag
809 	if (ln.length < 2 || (!ln[1].isAlpha && ln[1] != '#' && ln[1] != '/')) {
810 		// found no match, return escaped '<'
811 		logTrace("Found stray '<' in DDOC string.");
812 		ln.popFront();
813 		return "&lt;";
814 	}
815 
816 	// skip over regular start/end tag
817 	auto idx = ln.indexOf(">");
818 	if (idx < 0) {
819 		ln.popFront();
820 		return "<";
821 	}
822 	auto ret = ln[0 .. idx+1];
823 	ln = ln[ret.length .. $];
824 	return ret;
825 }
826 
827 private string skipUrl(ref string ln)
828 {
829 	if( !ln.startsWith("http://") && !ln.startsWith("http://") )
830 		return null;
831 
832 	bool saw_dot = false;
833 	size_t i = 7;
834 
835 	for_loop:
836 	while( i < ln.length ){
837 		switch( ln[i] ){
838 			default:
839 				break for_loop;
840 			case 'a': .. case 'z':
841 			case 'A': .. case 'Z':
842 			case '0': .. case '9':
843 			case '_', '-', '?', '=', '%', '&', '/', '+', '#', '~':
844 				break;
845 			case '.':
846 				saw_dot = true;
847 				break;
848 		}
849 		i++;
850 	}
851 
852 	if( saw_dot ){
853 		auto ret = ln[0 .. i];
854 		ln = ln[i .. $];
855 		return ret;
856 	} else return null;
857 }
858 
859 private string skipIdent(ref string str)
860 {
861 	string strcopy = str;
862 
863 	if (str.length >= 2 && str[0] == '.' && (str[1].isAlpha || str[1] == '_'))
864 		str.popFront();
865 
866 	bool last_was_ident = false;
867 	while( !str.empty ){
868 		auto ch = str.front;
869 
870 		if( last_was_ident ){
871 			// dots are allowed if surrounded by identifiers
872 			if( ch == '.' ) last_was_ident = false;
873 			else if( ch != '_' && (ch < '0' || ch > '9') && !std.uni.isAlpha(ch) ) break;
874 		} else {
875 			if( ch != '_' && !std.uni.isAlpha(ch) ) break;
876 			last_was_ident = true;
877 		}
878 		str.popFront();
879 	}
880 
881 	// if the identifier ended in a '.', remove it again
882 	if( str.length != strcopy.length && !last_was_ident )
883 		str = strcopy[strcopy.length-str.length-1 .. $];
884 
885 	return strcopy[0 .. strcopy.length-str.length];
886 }
887 
888 private bool isIdent(string str)
889 {
890 	skipIdent(str);
891 	return str.length == 0;
892 }
893 
894 private void parseMacros(ref string[string] macros, in string[] lines)
895 {
896 	string name;
897 	foreach (string ln; lines) {
898 		// macro definitions are of the form IDENT = ...
899 		auto pidx = ln.indexOf('=');
900 		if (pidx > 0) {
901 			auto tmpnam = ln[0 .. pidx].strip();
902 			// got new macro definition?
903 			if (isIdent(tmpnam)) {
904 
905 				// strip the previous macro
906 				if (name.length) macros[name] = macros[name].stripDD();
907 
908 				// start parsing the new macro
909 				name = tmpnam;
910 				macros[name] = stripLeftDD(ln[pidx+1 .. $]);
911 				continue;
912 			}
913 		}
914 
915 		// append to previous macro definition, if any
916 		macros[name] ~= "\n" ~ ln;
917 	}
918 }
919 
920 private int baseIndent(string[] lines)
921 {
922 	if( lines.length == 0 ) return 0;
923 	int ret = int.max;
924 	foreach( ln; lines ){
925 		int i = 0;
926 		while( i < ln.length && (ln[i] == ' ' || ln[i] == '\t') )
927 			i++;
928 		if( i < ln.length ) ret = min(ret, i);
929 	}
930 	return ret;
931 }
932 
933 private string unindent(string ln, int amount)
934 {
935 	while( amount > 0 && ln.length > 0 && (ln[0] == ' ' || ln[0] == '\t') )
936 		ln = ln[1 .. $], amount--;
937 	return ln;
938 }
939 
940 private string stripLeftDD(string s)
941 {
942 	while (!s.empty && (s.front == ' ' || s.front == '\t' || s.front == '\r' || s.front == '\n'))
943 		s.popFront();
944 	return s;
945 }
946 
947 private string specialStrip(string s)
948 {
949 	import std.algorithm : among;
950 
951 	// strip trailing whitespace for all lines but the last
952 	size_t idx = 0;
953 	while (true) {
954 		auto nidx = s[idx .. $].indexOf('\n');
955 		if (nidx < 0) break;
956 		nidx += idx;
957 		auto strippedfront = s[0 .. nidx].stripRightDD();
958 		s = strippedfront ~ "\n" ~ s[nidx+1 .. $];
959 		idx = strippedfront.length + 1;
960 	}
961 
962 	// strip the first character, if whitespace
963 	if (!s.empty && s.front.among!(' ', '\t', '\n', '\r')) s.popFront();
964 
965 	return s;
966 }
967 
968 private string stripRightDD(string s)
969 {
970 	while (!s.empty && (s.back == ' ' || s.back == '\t' || s.back == '\r' || s.back == '\n'))
971 		s.popBack();
972 	return s;
973 }
974 
975 private string stripDD(string s)
976 {
977 	return s.stripLeftDD.stripRightDD;
978 }
979 
980 import std.stdio;
981 unittest {
982 	auto src = "$(M a b)\n$(M a\nb)\nMacros:\n	M =     -$0-\n";
983 	auto dst = "-a b-\n-a\nb-\n";
984 	assert(formatDdocComment(src) == dst);
985 }
986 
987 unittest {
988 	auto src = "\n  $(M a b)\n$(M a  \nb)\nMacros:\n	M =     -$0-  \n\nN=$0";
989 	auto dst = "-a b-\n-a\nb-\n";
990 	assert(formatDdocComment(src) == dst);
991 }
992 
993 unittest {
994 	auto src = "$(M a, b)\n$(M a,\n    b)\nMacros:\n	M = -$1-\n\n	+$2+\n\n	N=$0";
995 	auto dst = "-a-\n\n	+b+\n-a-\n\n	+    b+\n";
996 	assert(formatDdocComment(src) == dst);
997 }
998 
999 unittest {
1000 	auto src = "$(GLOSSARY a\nb)\nMacros:\n	GLOSSARY = $(LINK2 glossary.html#$0, $0)";
1001 	auto dst = "<a href=\"glossary.html#a\nb\">a\nb</a>\n";
1002 	assert(formatDdocComment(src) == dst);
1003 }
1004 
1005 unittest {
1006 	auto src = "a > b < < c > <a <# </ <br> <abc> <.abc> <-abc> <+abc> <0abc> <abc-> <> <!-- c --> <!--> <! > <!-- > >a.";
1007 	auto dst = "a &gt; b &lt; &lt; c &gt; <a <# </ <br> <abc> &lt;.abc&gt; &lt;-abc&gt; &lt;+abc&gt; &lt;0abc&gt; <abc-> &lt;&gt; <!-- c --> &lt;!--&gt; &lt;! &gt; &lt;!-- &gt; &gt;a.\n";
1008 	assert(formatDdocComment(src) == dst);
1009 }
1010 
1011 unittest {
1012 	auto src = "& &a &lt; &#lt; &- &03; &;";
1013 	auto dst = "&amp; &a &lt; &#lt; &amp;- &amp;03; &amp;;\n";
1014 	assert(formatDdocComment(src) == dst);
1015 }
1016 
1017 unittest {
1018 	auto src = "<a href=\"abc\">test $(LT)peter@parker.com$(GT)</a>\nMacros:\nLT = &lt;\nGT = &gt;";
1019 	auto dst = "<a href=\"abc\">test &lt;peter@parker.com&gt;</a>\n";
1020 //writeln(formatDdocComment(src).splitLines().map!(s => "|"~s~"|").join("\n"));
1021 	assert(formatDdocComment(src) == dst);
1022 }
1023 
1024 unittest {
1025 	auto src = "$(LIX a, b, c, d)\nMacros:\nLI = [$0]\nLIX = $(LI $1)$(LIX $+)";
1026 	auto dst = "[a][b][c][d]\n";
1027 	assert(formatDdocComment(src) == dst);
1028 }
1029 
1030 unittest {
1031 	auto src = "Testing `inline <code>`.";
1032 	auto dst = "Testing <code class=\"lang-d\">inline &lt;code&gt;</code>.\n";
1033 	assert(formatDdocComment(src) == dst);
1034 }
1035 
1036 unittest {
1037 	auto src = "Testing `inline $(CODE)`.";
1038 	auto dst = "Testing <code class=\"lang-d\">inline $(CODE)</code>.\n";
1039 	assert(formatDdocComment(src));
1040 }
1041 
1042 unittest {
1043 	auto src = "---\nthis is a `string`.\n---";
1044 	auto dst = "<section><pre class=\"code\"><code class=\"lang-d\"><span class=\"kwd\">this is </span><span class=\"pln\">a </span><span class=\"str\">`string`<wbr/></span><span class=\"pun\">.</span></code></pre>\n</section>\n";
1045 	assert(formatDdocComment(src) == dst);
1046 }
1047 
1048 unittest { // test for properly removed indentation in code blocks
1049 	auto src = "  ---\n  testing\n  ---";
1050 	auto dst = "<section><pre class=\"code\"><code class=\"lang-d\"><span class=\"pln\">testing</span></code></pre>\n</section>\n";
1051 	assert(formatDdocComment(src) == dst);
1052 }
1053 
1054 unittest { // issue #99 - parse macros in parameter sections
1055 	import std.algorithm : find;
1056 	auto src = "Params:\n\tfoo = $(B bar)";
1057 	auto dst = "<td> <b>bar</b></td></tr>\n</table>\n</section>\n";
1058 	assert(formatDdocComment(src).find("<td> ") == dst);
1059 }
1060 
1061 unittest { // issue #89 (minimal test) - empty first parameter
1062 	auto src = "$(DIV , foo)\nMacros:\nDIV=<div $1>$+</div>";
1063 	auto dst = "<div >foo</div>\n";
1064 	assert(formatDdocComment(src) == dst);
1065 }
1066 
1067 unittest { // issue #89 (complex test)
1068 	auto src =
1069 `$(LIST
1070 $(DIV oops,
1071 foo
1072 ),
1073 $(DIV ,
1074 bar
1075 ))
1076 Macros:
1077 LIST=$(UL $(LIX $1, $+))
1078 LIX=$(LI $1)$(LIX $+)
1079 UL=$(T ul, $0)
1080 LI = $(T li, $0)
1081 DIV=<div $1>$+</div>
1082 T=<$1>$+</$1>
1083 `;
1084 	auto dst = "<ul><li><div oops>foo\n</div></li><li><div >bar\n</div></li></ul>\n";
1085 	assert(formatDdocComment(src) == dst);
1086 }
1087 
1088 unittest { // issue #95 - trailing newlines must be stripped in macro definitions
1089 	auto src = "$(FOO)\nMacros:\nFOO=foo\n\nBAR=bar";
1090 	auto dst = "foo\n";
1091 	assert(formatDdocComment(src) == dst);
1092 }
1093 
1094 unittest { // missing macro closing clamp (because it's in a different section)
1095 	auto src = "$(B\n\n)";
1096 	auto dst = "(B\n<section><p>)\n</p>\n</section>\n";
1097 	assert(formatDdocComment(src) == dst);
1098 }
1099 
1100 unittest { // closing clamp should be found in a different *paragraph* of the same section, though
1101 	auto src = "foo\n\n$(B\n\n)";
1102 	auto dst = "foo\n<section><p><b></b>\n</p>\n</section>\n";
1103 	assert(formatDdocComment(src) == dst);
1104 }
1105 
1106 unittest { // more whitespace testing
1107 	auto src = "$(M    a   ,   b   ,   c   )\nMacros:\nM =    A$0B$1C$2D$+E";
1108     auto dst = "A   a   ,   b   ,   c   B   a   C  b   D  b   ,   c   E\n";
1109     assert(formatDdocComment(src) == dst);
1110 }
1111 
1112 unittest { // more whitespace testing
1113 	auto src = "  $(M  \n  a  \n  ,  \n  b \n  ,  \n  c  \n  )  \nMacros:\nM =    A$0B$1C$2D$+E";
1114     auto dst = "A  a\n  ,\n  b\n  ,\n  c\n  B  a\n  C  b\n  D  b\n  ,\n  c\n  E\n";
1115     assert(formatDdocComment(src) == dst);
1116 }
1117 
1118 unittest { // escape in backtick code
1119 	auto src = "`<b>&amp;`";
1120 	auto dst = "<code class=\"lang-d\">&lt;b&gt;&amp;amp;</code>\n";
1121 	assert(formatDdocComment(src) == dst);
1122 }
1123 
1124 unittest { // escape in code blocks
1125 	auto src = "---\n<b>&amp;\n---";
1126 	auto dst = "<section><pre class=\"code\"><code class=\"lang-d\"><span class=\"pun\">&lt;</span><span class=\"pln\">b</span><span class=\"pun\">&gt;&amp;</span><span class=\"pln\">amp</span><span class=\"pun\">;</span></code></pre>\n</section>\n";
1127 	assert(formatDdocComment(src) == dst);
1128 }
1129 
1130 unittest { // #81 empty first macro arguments
1131 	auto src = "$(BOOKTABLE,\ntest)\nMacros:\nBOOKTABLE=<table $1>$+</table>";
1132 	auto dst = "<table >test</table>\n";
1133 	assert(formatDdocComment(src) == dst);
1134 }
1135 
1136 unittest { // #117 underscore identifiers as macro param
1137 	auto src = "$(M __foo) __foo `__foo` $(D_CODE __foo)\nMacros:\nM=http://$1.com";
1138 	auto dst = "http://_foo.com _foo <code class=\"lang-d\">__foo</code> <pre class=\"d_code\">_foo</pre>\n";
1139 	assert(formatDdocComment(src) == dst);
1140 }
1141 
1142 unittest { // #109 dot followed by unicode character causes infinite loop
1143 	auto src = ".”";
1144 	auto dst = ".”\n";
1145 	assert(formatDdocComment(src) == dst, [formatDdocComment(src)].to!string);
1146 }