1 /**
2 	Parses DMD JSON output and builds up a documentation syntax tree (JSON format from DMD 2.063.2).
3 
4 	Copyright: © 2012-2015 RejectedSoftware e.K.
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig
7 */
8 module ddox.parsers.jsonparser;
9 
10 import ddox.ddox;
11 import ddox.entities;
12 
13 import std.algorithm;
14 import std.conv;
15 import std.exception;
16 import std.range;
17 import std.stdio;
18 import std.string;
19 import std.typecons;
20 import core.demangle;
21 import vibe.core.log;
22 import vibe.data.json;
23 
24 
25 Package parseJsonDocs(Json json, Package root = null)
26 {
27 	if( !root ) root = new Package(null, null);
28 	Parser p;
29 	foreach( mod; json ){
30 		p.parseModule(mod, root);
31 	}
32 	p.resolveTypes(root);
33 	return root;
34 }
35 
36 private struct Parser
37 {
38 	private Tuple!(Type, Entity)[] m_primTypes;
39 	private Declaration[string] m_typeMap;
40 
41 	void resolveTypes(Package root)
42 	{
43 		bool isTypeDecl(Declaration a)
44 		{
45 			switch(a.kind){
46 				default: return false;
47 				case DeclarationKind.Struct:
48 				case DeclarationKind.Union:
49 				case DeclarationKind.Class:
50 				case DeclarationKind.Interface:
51 				case DeclarationKind.Enum:
52 					return true;
53 				case DeclarationKind.Alias:
54 					return (cast(AliasDeclaration)a).targetType !is null;
55 				case DeclarationKind.TemplateParameter:
56 					return true;
57 				case DeclarationKind.Template:
58 					// support eponymous template types
59 					auto td = cast(TemplateDeclaration)a;
60 					// be optimistic for templates without content that they are in fact types
61 					if (!td.members.length) return true;
62 					// otherwise require an actual eponymous type member
63 					auto mi = td.members.countUntil!(m => m.name == a.name);
64 					return mi >= 0 && isTypeDecl(td.members[mi]);
65 			}
66 		}
67 
68 		foreach (t; m_primTypes) {
69 			auto decl = t[1].lookup!Declaration(t[0].typeName);
70 			if (!decl || !isTypeDecl(decl)) {
71 				auto pd = t[0].typeName in m_typeMap;
72 				if (pd) decl = *pd;
73 			}
74 			if (decl && isTypeDecl(decl))
75 				t[0].typeDecl = decl;
76 		}
77 
78 		// fixup class bases
79 		root.visit!ClassDeclaration((decl){
80 			if( decl.baseClass && decl.baseClass.typeDecl && !cast(ClassDeclaration)decl.baseClass.typeDecl )
81 				decl.baseClass = null;
82 			foreach( i; decl.derivedInterfaces )
83 				if( i.typeDecl && !cast(InterfaceDeclaration)i.typeDecl )
84 					i.typeDecl = null;
85 			assert(decl);
86 		});
87 
88 		// fixup interface bases
89 		root.visit!InterfaceDeclaration((decl){
90 			foreach( i; decl.derivedInterfaces )
91 				if( i.typeDecl && !cast(InterfaceDeclaration)i.typeDecl )
92 					i.typeDecl = null;
93 			assert(decl);
94 		});
95 	}
96 
97 	void parseModule(Json json, Package root_package)
98 	{
99 		Module mod;
100 		if( "name" !in json ){
101 			logError("No name attribute in module %s - ignoring", json.filename.opt!string);
102 			return;
103 		}
104 		auto path = json.name.get!string.split(".");
105 		Package p = root_package;
106 		foreach( i, pe; path ){
107 			if( i+1 < path.length ) p = p.getOrAddPackage(pe);
108 			else mod = p.createModule(pe);
109 		}
110 
111 		mod.file = json.file.get!string;
112 		mod.docGroup = new DocGroup(mod, json.comment.opt!string());
113 		mod.members = parseDeclList(json.members, mod);
114 	}
115 
116 	Declaration[] parseDeclList(Json json, Entity parent)
117 	{
118 		if( json.type == Json.Type.Undefined ) return null;
119 		DocGroup lastdoc;
120 		Declaration[] ret;
121 		foreach( mem; json ){
122 			auto decl = parseDecl(mem, parent);
123 			if( !decl ) continue;
124 			auto doc = decl.docGroup;
125 			if( lastdoc && (doc.text == lastdoc.text && doc.text.length || doc.comment.isDitto) ){
126 				lastdoc.members ~= decl;
127 				decl.docGroup = lastdoc;
128 			} else if( doc.comment.isPrivate ){
129 				decl.protection = Protection.Private;
130 				lastdoc = null;
131 			} else lastdoc = decl.docGroup;
132 			ret ~= decl;
133 		}
134 		return ret;
135 	}
136 
137 	Declaration parseDecl(Json json, Entity parent)
138 	{
139 		Declaration ret;
140 
141 		// DMD outputs templates with the wrong kind sometimes
142 		if( json.name.get!string().canFind('(') ){
143 			ret = parseTemplateDecl(json, parent);
144 		} else {
145 			switch( json.kind.get!string ){
146 				default:
147 					logWarn("Unknown declaration kind: %s", json.kind.get!string);
148 					return null;
149 				case "import":
150 				case "static import":
151 					// TODO: use for symbol resolving
152 					return null;
153 				case "destructor": return null;
154 				case "mixin": return null; // TODO: support documented mixins
155 				case "alias":
156 					ret = parseAliasDecl(json, parent);
157 					break;
158 				case "function":
159 				case "allocator":
160 				case "deallocator":
161 				case "constructor":
162 					ret = parseFunctionDecl(json, parent);
163 					break;
164 				case "enum":
165 					ret = parseEnumDecl(json, parent);
166 					break;
167 				case "enum member":
168 					ret = parseEnumMemberDecl(json, parent);
169 					break;
170 				case "struct":
171 				case "union":
172 				case "class":
173 				case "interface":
174 					ret = parseCompositeDecl(json, parent);
175 					break;
176 				case "variable":
177 					ret = parseVariableDecl(json, parent);
178 					break;
179 				case "template":
180 					ret = parseTemplateDecl(json, parent);
181 					break;
182 			}
183 		}
184 
185 		ret.protection = parseProtection(json.protection);
186 		ret.line = json["line"].opt!int;
187 		ret.docGroup = new DocGroup(ret, json.comment.opt!string());
188 
189 		return ret;
190 	}
191 
192 	auto parseAliasDecl(Json json, Entity parent)
193 	{
194 		auto ret = new AliasDeclaration(parent, json.name.get!string);
195 		ret.attributes = json.storageClass.opt!(Json[]).map!(j => j.get!string).array;
196 		ret.targetType = parseType(json, ret, null);
197 		if( ret.targetType && ret.targetType.kind == TypeKind.Primitive && ret.targetType.typeName.length == 0 )
198 			ret.targetType = null;
199 		insertIntoTypeMap(ret);
200 		return ret;
201 	}
202 
203 	auto parseFunctionDecl(Json json, Entity parent)
204 	{
205 		auto ret = new FunctionDeclaration(parent, json.name.opt!string);
206 		ret.type = parseType(json, ret, "void()");
207 		// TODO: use "storageClass" and "parameters" fields
208 		if( ret.type.kind == TypeKind.Function ){
209 			ret.returnType = ret.type.returnType;
210 			ret.attributes = ret.type.attributes ~ ret.type.modifiers;
211 			if (auto psc = "storageClass" in json)
212 				foreach (sc; *psc)
213 					if (!ret.attributes.canFind(sc.get!string))
214 						ret.attributes ~= sc.get!string;
215 
216 			auto params = json.parameters.opt!(Json[]);
217 			if (!params) {
218 				params.length = ret.type.parameterTypes.length;
219 				foreach (i, pt; ret.type.parameterTypes) {
220 					auto jp = Json.emptyObject;
221 					jp.name = ret.type._parameterNames[i];
222 					jp["type"] = pt.text;
223 					if (ret.type._parameterDefaultValues[i])
224 						jp["default"] = ret.type._parameterDefaultValues[i].valueString;
225 					params[i] = jp;
226 				}
227 			}
228 
229 			foreach (i, p; params) {
230 				auto pname = p.name.opt!string;
231 				auto decl = new VariableDeclaration(ret, pname);
232 				foreach (sc; p["storageClass"].opt!(Json[]))
233 					if (!decl.attributes.canFind(sc.get!string))
234 						decl.attributes ~= sc.get!string;
235 				decl.type = parseType(p, ret);
236 				if (auto pdv = "default" in p)
237 					decl.initializer = parseValue(pdv.opt!string);
238 				ret.parameters ~= decl;
239 			}
240 		} else {
241 			logError("Expected function type for '%s'/'%s', got %s %s", json["type"].opt!string, demangleType(json["deco"].opt!string), ret.type.kind, ret.type.typeName);
242 		}
243 		return ret;
244 	}
245 
246 	auto parseEnumDecl(Json json, Entity parent)
247 	{
248 		auto ret = new EnumDeclaration(parent, json.name.get!string);
249 		insertIntoTypeMap(ret);
250 		if( "base" !in json ){ // FIXME: parse deco instead
251 			if( auto pd = "baseDeco" in json )
252 				json.base = demanglePrettyType(pd.get!string());
253 		}
254 		ret.baseType = parseType(json.base, ret);
255 		auto mems = parseDeclList(json.members, ret);
256 		foreach( m; mems ){
257 			auto em = cast(EnumMemberDeclaration)m;
258 			assert(em !is null, "Enum containing non-enum-members?");
259 			ret.members ~= em;
260 		}
261 		return ret;
262 	}
263 
264 	auto parseEnumMemberDecl(Json json, Entity parent)
265 	{
266 		auto ret = new EnumMemberDeclaration(parent, json.name.get!string);
267 		if (json.value.opt!string.length)
268 			ret.value = parseValue(json.value.opt!string);
269 		return ret;
270 	}
271 
272 	auto parseCompositeDecl(Json json, Entity parent)
273 	{
274 		CompositeTypeDeclaration ret;
275 		switch(json.kind.get!string){
276 			default:
277 				logWarn("Invalid composite decl kind: %s", json.kind.get!string);
278 				return new StructDeclaration(parent, json.name.get!string);
279 			case "struct":
280 				ret = new StructDeclaration(parent, json.name.get!string);
281 				break;
282 			case "union":
283 				ret = new UnionDeclaration(parent, json.name.get!string);
284 				break;
285 			case "class":
286 				auto clsdecl = new ClassDeclaration(parent, json.name.get!string);
287 				if( clsdecl.qualifiedName != "object.Object" )
288 					clsdecl.baseClass = parseType(json.base, clsdecl, "Object", false);
289 				foreach( intf; json.interfaces.opt!(Json[]) )
290 					clsdecl.derivedInterfaces ~= parseType(intf, clsdecl);
291 				ret = clsdecl;
292 				break;
293 			case "interface":
294 				auto intfdecl = new InterfaceDeclaration(parent, json.name.get!string);
295 				foreach( intf; json.interfaces.opt!(Json[]) )
296 					intfdecl.derivedInterfaces ~= parseType(intf, intfdecl);
297 				ret = intfdecl;
298 				break;
299 		}
300 
301 		insertIntoTypeMap(ret);
302 
303 		ret.members = parseDeclList(json.members, ret);
304 
305 		return ret;
306 	}
307 
308 	Declaration parseVariableDecl(Json json, Entity parent)
309 	{
310 		if (json.storageClass.opt!(Json[]).canFind!(j => j.opt!string == "enum")) {
311 			auto ret = new EnumMemberDeclaration(parent, json.name.get!string);
312 			if (json["init"].opt!string.length)
313 				ret.value = parseValue(json["init"].opt!string);
314 			return ret;
315 		} else {
316 			auto ret = new VariableDeclaration(parent, json.name.get!string);
317 			ret.type = parseType(json, ret);
318 			if (json["init"].opt!string.length)
319 				ret.initializer = parseValue(json["init"].opt!string);
320 			return ret;
321 		}
322 	}
323 
324 	auto parseTemplateDecl(Json json, Entity parent)
325 	{
326 		auto ret = new TemplateDeclaration(parent, json.name.get!string);
327 		foreach (arg; json.parameters.opt!(Json[])) {
328 			string argstr;
329 			switch (arg.kind.get!string) {
330 				case "value":
331 					if (auto pt = "type" in arg) argstr = pt.get!string ~ ' ';
332 					else argstr = demanglePrettyType(arg.deco.get!string) ~ ' ';
333 					goto default;
334 				case "alias":
335 					argstr = "alias ";
336 					goto default;
337 				case "tuple":
338 					argstr ~= arg.name.get!string ~ "...";
339 					break;
340 				default:
341 					argstr ~= arg.name.get!string;
342 			}
343 			ret.templateArgs ~= new TemplateParameterDeclaration(ret, argstr);
344 			ret.templateConstraint = json.constraint.opt!string;
345 		}
346 		ret.members = parseDeclList(json.members, ret);
347 		return ret;
348 	}
349 
350 	Type parseType(Json json, Entity sc, string def_type = "void", bool warn_id_not_exists = true)
351 	{
352 		string str;
353 		if( json.type == Json.Type.Undefined ){
354 			if (warn_id_not_exists) logWarn("No type found for %s.", sc.qualifiedName);
355 			str = def_type;
356 		} else if( json.type == Json.Type.String ) str = json.get!string();
357 		else if( auto pv = "deco" in json ) str = demanglePrettyType(pv.get!string());
358 		else if( auto pv = "type" in json ) str = pv.get!string();
359 		else if( auto pv = "originalType" in json ) str = pv.get!string();
360 
361 		if( str.length == 0 ) str = def_type;
362 
363 		if( !str.length ) return null;
364 
365 		auto tokens = tokenizeDSource(str);
366 		
367 		logDebug("parse type '%s'", str);
368 		try {
369 			auto type = parseTypeDecl(tokens, sc);
370 			type.text = str;
371 			return type;
372 		} catch( Exception e ){
373 			logError("Error parsing type '%s': %s", str, e.msg);
374 			auto type = new Type;
375 			type.text = str;
376 			type.typeName = str;
377 			type.kind = TypeKind.Primitive;
378 			return type;
379 		}
380 	}
381 
382 	Value parseValue(string str)
383 	{
384 		auto ret = new Value;
385 		//ret.type = ;
386 		ret.valueString = str;
387 		return ret;
388 	}
389 
390 	Protection parseProtection(Json prot)
391 	{
392 		switch( prot.opt!string ){
393 			default: return Protection.Public;
394 			case "package": return Protection.Package;
395 			case "protected": return Protection.Protected;
396 			case "private": return Protection.Private;
397 		}
398 	}
399 
400 	Declaration lookupDecl(string qualified_name, Entity sc)
401 	{
402 		while(sc){
403 			auto ent = cast(Declaration)sc.lookup(qualified_name);
404 			if( ent ) return ent;
405 			sc = sc.parent;
406 		}
407 		return null;
408 	}
409 
410 	Type parseTypeDecl(ref string[] tokens, Entity sc)
411 	{
412 
413 		auto ret = parseType(tokens, sc);
414 		return ret;
415 	}
416 
417 	Type parseType(ref string[] tokens, Entity sc)
418 	{
419 		string[] attributes;
420 		auto basic_type = parseBasicType(tokens, sc, attributes);
421 		basic_type.attributes ~= attributes;
422 		return basic_type;	
423 	}
424 
425 	Type parseBasicType(ref string[] tokens, Entity sc, out string[] attributes)
426 	{
427 		static immutable global_attribute_keywords = ["abstract", "auto", "const", "deprecated", "enum",
428 			"extern", "final", "immutable", "inout", "shared", "nothrow", "override", "pure",
429 			"__gshared", "scope", "static", "synchronize"];
430 
431 		static immutable parameter_attribute_keywords = ["auto", "const", "final", "immutable", "in", "inout",
432 			"lazy", "out", "ref", "scope", "shared"];
433 
434 		static immutable member_function_attribute_keywords = ["const", "immutable", "inout", "shared", "pure", "nothrow"];
435 		
436 			
437 		if( tokens.length > 0 && tokens[0] == "extern" ){
438 			enforce(tokens[1] == "(");
439 			enforce(tokens[3] == ")");
440 			attributes ~= join(tokens[0 .. 4]);
441 			tokens = tokens[4 .. $];
442 		}
443 		
444 		immutable string[] attribute_keywords = global_attribute_keywords ~ parameter_attribute_keywords ~ member_function_attribute_keywords;
445 		/*final switch( sc ){
446 			case DeclScope.Global: attribute_keywords = global_attribute_keywords; break;
447 			case DeclScope.Parameter: attribute_keywords = parameter_attribute_keywords; break;
448 			case DeclScope.Class: attribute_keywords = member_function_attribute_keywords; break;
449 		}*/
450 
451 		void parseAttributes(ref string[] dst, const(string)[] keywords)
452 		{
453 			while( tokens.length > 0 ){
454 				if( tokens.front == "@" ){
455 					tokens.popFront();
456 					dst ~= "@"~tokens.front;
457 					tokens.popFront();
458 				} else if( keywords.countUntil(tokens[0]) >= 0 && tokens[1] != "(" ){
459 					dst ~= tokens.front;
460 					tokens.popFront();
461 				} else break;
462 			}
463 		}
464 
465 		parseAttributes(attributes, attribute_keywords);
466 
467 
468 		Type type;
469 		static immutable const_modifiers = ["const", "immutable", "shared", "inout"];
470 		if (tokens.length > 2 && tokens[1] == "(" && const_modifiers.countUntil(tokens[0]) >= 0) {
471 			auto mod = tokens.front;
472 			tokens.popFrontN(2);
473 			string[] subattrs;
474 			type = parseBasicType(tokens, sc, subattrs);
475 			type.modifiers ~= mod;
476 			type.attributes ~= subattrs;
477 			enforce(!tokens.empty && tokens.front == ")", format("Missing ')' for '%s('", mod));
478 			tokens.popFront();
479 		} else {
480 			type = new Type;
481 			type.kind = TypeKind.Primitive;
482 			m_primTypes ~= tuple(type, sc);
483 
484 			size_t start = 0, end;
485 			if( tokens[start] == "." ) start++;
486 			for( end = start; end < tokens.length && isIdent(tokens[end]); ){
487 				end++;
488 				if( end >= tokens.length || tokens[end] != "." )
489 					break;
490 				end++;
491 			}
492 
493 			size_t i = end;
494 
495 			string type_name, nested_name;
496 			if( i == 0 && tokens[0] == "..." ){
497 				type_name = "...";
498 				nested_name = null;
499 			} else if( i == 0 && tokens[0] == "(" ){
500 				type_name = "constructor";
501 				nested_name = null;
502 			} else {
503 				enforce(i > 0, "Expected identifier but got "~tokens.front);
504 				type.typeName = join(tokens[start .. end]);
505 				//type.typeDecl = cast(Declaration)sc.lookup(type.typeName);
506 				tokens.popFrontN(i);
507 				
508 				if (type.typeName == "typeof" && !tokens.empty && tokens.front == "(") {
509 					type.typeName ~= "(";
510 					tokens.popFront();
511 					int level = 1;
512 					while (!tokens.empty && level > 0) {
513 						if (tokens.front == "(") level++;
514 						else if( tokens.front == ")") level--;
515 						type.typeName ~= tokens.front;
516 						tokens.popFront();
517 					}
518 				} else if( !tokens.empty && tokens.front == "!" ){
519 					tokens.popFront();
520 					if( tokens.front == "(" ){
521 						size_t j = 1;
522 						int cc = 1;
523 						while( cc > 0 ){
524 							assert(j < tokens.length);
525 							if( tokens[j] == "(" ) cc++;
526 							else if( tokens[j] == ")") cc--;
527 							j++;
528 						}
529 						type.templateArgs = join(tokens[0 .. j]);
530 						tokens.popFrontN(j);
531 						logDebug("templargs: %s", type.templateArgs);
532 					} else {
533 						type.templateArgs = tokens[0];
534 						tokens.popFront();
535 					}
536 					
537 					// HACK: dropping the actual type name here!
538 					while (!tokens.empty && tokens.front == ".") {
539 						tokens.popFront();
540 						if (!tokens.empty()) tokens.popFront();
541 					}
542 				}
543 			}
544 		}
545 		
546 		while( !tokens.empty ){
547 			if( tokens.front == "*" ){
548 				auto ptr = new Type;
549 				ptr.kind = TypeKind.Pointer;
550 				ptr.elementType = type;
551 				type = ptr;
552 				tokens.popFront();
553 			} else if( tokens.front == "[" ){
554 				tokens.popFront();
555 				if( tokens.front == "]" ){
556 					auto arr = new Type;
557 					arr.kind = TypeKind.Array;
558 					arr.elementType = type;
559 					type = arr;
560 				} else {
561 					string[] tokens_copy = tokens;
562 					Type keytp;
563 					if (!isDigit(tokens.front[0]) && tokens.front != "!") keytp = parseType(tokens_copy, sc);
564 					if (keytp && !tokens_copy.empty && tokens_copy.front == "]") {
565 						tokens = tokens_copy;
566 						logDebug("GOT TYPE: %s", keytp.toString());
567 						auto aa = new Type;
568 						aa.kind = TypeKind.AssociativeArray;
569 						aa.elementType = type;
570 						aa.keyType = keytp;
571 						type = aa;
572 					} else {
573 						auto arr = new Type;
574 						arr.kind = TypeKind.StaticArray;
575 						arr.elementType = type;
576 						arr.arrayLength = tokens.front;
577 						tokens.popFront();
578 						while (!tokens.empty && tokens.front != "]") {
579 							arr.arrayLength ~= tokens.front;
580 							tokens.popFront();
581 						}
582 						type = arr;
583 					}
584 				}
585 				enforce(tokens.front == "]", "Expected ']', got '"~tokens.front~"'.");
586 				tokens.popFront();
587 			} else break;
588 		}
589 		
590 		while (!tokens.empty && (tokens.front == "function" || tokens.front == "delegate" || tokens.front == "(")) {
591 			Type ftype = new Type;
592 			ftype.kind = tokens.front == "(" || tokens.front == "function" ? TypeKind.Function : TypeKind.Delegate;
593 			ftype.returnType = type;
594 			if (tokens.front != "(") tokens.popFront();
595 			enforce(tokens.front == "(");
596 			tokens.popFront();
597 			if (!tokens.empty && tokens.front == ",") tokens.popFront(); // sometimes demangleType() returns something like "void(, ...)"
598 			while (true) {
599 				if (tokens.front == ")") break;
600 				enforce(!tokens.empty);
601 				ftype.parameterTypes ~= parseTypeDecl(tokens, sc);
602 				if (tokens.front != "," && tokens.front != ")") {
603 					ftype._parameterNames ~= tokens.front;
604 					tokens.popFront();
605 				} else ftype._parameterNames ~= null;
606 				if (tokens.front == "...") {
607 					ftype._parameterNames[$-1] ~= tokens.front;
608 					tokens.popFront();
609 				}
610 				if (tokens.front == "=") {
611 					tokens.popFront();
612 					string defval;
613 					int ccount = 0;
614 					while (!tokens.empty) {
615 						if (ccount == 0 && (tokens.front == "," || tokens.front == ")"))
616 							break;
617 						if (tokens.front == "(") ccount++;
618 						else if (tokens.front == ")") ccount--;
619 						defval ~= tokens.front;
620 						tokens.popFront();
621 					}
622 					ftype._parameterDefaultValues ~= parseValue(defval);
623 					logDebug("got defval %s", defval);
624 				} else ftype._parameterDefaultValues ~= null;
625 				if (tokens.front == ")") break;
626 				enforce(tokens.front == ",", "Expecting ',', got "~tokens.front);
627 				tokens.popFront();
628 			}
629 			tokens.popFront();
630 
631 			parseAttributes(ftype.attributes, member_function_attribute_keywords);
632 
633 			type = ftype;
634 		}
635 
636 		return type;
637 	}
638 	
639 	string[] tokenizeDSource(string dsource_)
640 	{
641 		static immutable dstring[] tokens = [
642 			"/", "/=", ".", "..", "...", "&", "&=", "&&", "|", "|=", "||",
643 			"-", "-=", "--", "+", "+=", "++", "<", "<=", "<<", "<<=",
644 			"<>", "<>=", ">", ">=", ">>=", ">>>=", ">>", ">>>", "!", "!=",
645 			"!<>", "!<>=", "!<", "!<=", "!>", "!>=", "(", ")", "[", "]",
646 			"{", "}", "?", ",", ";", ":", "$", "=", "==", "*", "*=",
647 			"%", "%=", "^", "^=", "~", "~=", "@", "=>", "#"
648 		];
649 		static bool[dstring] token_map;
650 		
651 		if( !token_map.length ){
652 			foreach( t; tokens )
653 				token_map[t] = true;
654 			token_map.rehash;
655 		}
656 		
657 		dstring dsource = to!dstring(dsource_);
658 		
659 		dstring[] ret;
660 		outer:
661 		while(true){
662 			dsource = stripLeft(dsource);
663 			if( dsource.length == 0 ) break;
664 			
665 			// special token?
666 			foreach_reverse( i; 1 .. min(5, dsource.length+1) )
667 				if( dsource[0 .. i] in token_map ){
668 					ret ~= dsource[0 .. i];
669 					dsource.popFrontN(i);
670 					continue outer;
671 				}
672 			
673 			// identifier?
674 			if( dsource[0] == '_' || std.uni.isAlpha(dsource[0]) ){
675 				size_t i = 1;
676 				while( i < dsource.length && (dsource[i] == '_' || std.uni.isAlpha(dsource[i]) || isDigit(dsource[i])) ) i++;
677 				ret ~= dsource[0 .. i];
678 				dsource.popFrontN(i);
679 				continue;
680 			}
681 			
682 			// character literal?
683 			if( dsource[0] == '\'' ){
684 				size_t i = 1;
685 				while( dsource[i] != '\'' ){
686 					if( dsource[i] == '\\' ) i++;
687 					i++;
688 					enforce(i < dsource.length);
689 				}
690 				ret ~= dsource[0 .. i+1];
691 				dsource.popFrontN(i+1);
692 				continue;
693 			}
694 			
695 			// string? (incomplete!)
696 			if( dsource[0] == '"' ){
697 				size_t i = 1;
698 				while( dsource[i] != '"' ){
699 					if( dsource[i] == '\\' ) i++;
700 					i++;
701 					enforce(i < dsource.length);
702 				}
703 				ret ~= dsource[0 .. i+1];
704 				dsource.popFrontN(i+1);
705 				continue;
706 			}
707 			
708 			// number?
709 			if( isDigit(dsource[0]) || dsource[0] == '.' ){
710 				auto dscopy = dsource;
711 				parse!double(dscopy);
712 				ret ~= dsource[0 .. dsource.length-dscopy.length];
713 				dsource.popFrontN(dsource.length-dscopy.length);
714 				if( dsource.startsWith("u") ) dsource.popFront();
715 				else if( dsource.startsWith("f") ) dsource.popFront();
716 				continue;
717 			}
718 			
719 			ret ~= dsource[0 .. 1];
720 			dsource.popFront();
721 		}
722 		
723 		auto ret_ = new string[ret.length];
724 		foreach( i; 0 .. ret.length ) ret_[i] = to!string(ret[i]);
725 		return ret_;
726 	}
727 
728 	bool isDigit(dchar ch)
729 	{
730 		return ch >= '0' && ch <= '9';
731 	}
732 
733 	bool isIdent(string str)
734 	{
735 		if( str.length < 1 ) return false;
736 		foreach( i, dchar ch; str ){
737 			if( ch == '_' || std.uni.isAlpha(ch) ) continue;
738 			if( i > 0 && isDigit(ch) ) continue;
739 			return false;
740 		}
741 		return true;	
742 	}
743 
744 	string fullStrip(string s)
745 	{
746 		string chars = " \t\r\n";
747 		while( s.length > 0 && chars.countUntil(s[0]) >= 0 ) s.popFront();
748 		while( s.length > 0 && chars.countUntil(s[$-1]) >= 0 ) s.popBack();
749 		return s;
750 	}
751 
752 	void insertIntoTypeMap(Declaration decl)
753 	{
754 		string[] parts = split(decl.qualifiedName, ".");
755 		foreach( i; 0 .. parts.length ){
756 			auto partial_name = join(parts[i .. $], ".");
757 			m_typeMap[partial_name] = decl;
758 		}
759 	}
760 }
761 
762 string demanglePrettyType(string mangled_type)
763 {
764 	if (mangled_type == "n") return "typeof(null)"; // Workaround D issue 14410
765 	auto str = assumeUnique(demangleType(mangled_type));
766 	str = str.replace("immutable(char)[]", "string");
767 	str = str.replace("immutable(wchar)[]", "wstring");
768 	str = str.replace("immutable(dchar)[]", "dstring");
769 	return str;
770 }