1 /**
2 	Parses DMD JSON output and builds up a documentation syntax tree (JSON format from DMD 2.063.2).
3 
4 	Copyright: © 2012-2015 RejectedSoftware e.K.
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig
7 */
8 module ddox.parsers.jsonparser;
9 
10 import ddox.ddox;
11 import ddox.entities;
12 
13 import std.algorithm;
14 import std.conv;
15 import std.exception;
16 import std.range;
17 import std.stdio;
18 import std.string;
19 import std.typecons;
20 import core.demangle;
21 import vibe.core.log;
22 import vibe.data.json;
23 
24 
25 Package parseJsonDocs(Json json, Package root = null)
26 {
27 	if( !root ) root = new Package(null, null);
28 	Parser p;
29 	foreach( mod; json ){
30 		p.parseModule(mod, root);
31 	}
32 	p.resolveTypes(root);
33 	return root;
34 }
35 
36 private struct Parser
37 {
38 	private Tuple!(Type, Entity)[] m_primTypes;
39 	private Declaration[string] m_typeMap;
40 
41 	void resolveTypes(Package root)
42 	{
43 		bool isTypeDecl(Declaration a)
44 		{
45 			switch(a.kind){
46 				default: return false;
47 				case DeclarationKind.Struct:
48 				case DeclarationKind.Union:
49 				case DeclarationKind.Class:
50 				case DeclarationKind.Interface:
51 				case DeclarationKind.Enum:
52 					return true;
53 				case DeclarationKind.Alias:
54 					return (cast(AliasDeclaration)a).targetType !is null;
55 				case DeclarationKind.TemplateParameter:
56 					return true;
57 				case DeclarationKind.Template:
58 					// support eponymous template types
59 					auto td = cast(TemplateDeclaration)a;
60 					// be optimistic for templates without content that they are in fact types
61 					if (!td.members.length) return true;
62 					// otherwise require an actual eponymous type member
63 					auto mi = td.members.countUntil!(m => m.name == a.name);
64 					return mi >= 0 && isTypeDecl(td.members[mi]);
65 			}
66 		}
67 
68 		foreach (t; m_primTypes) {
69 			Declaration decl;
70 			if (t[0].typeName.length) decl = t[1].lookup!Declaration(t[0].typeName);
71 			if (!decl || !isTypeDecl(decl)) {
72 				auto pd = t[0].typeName in m_typeMap;
73 				if (pd) decl = *pd;
74 			}
75 			if (decl && isTypeDecl(decl))
76 				t[0].typeDecl = decl;
77 		}
78 
79 		// fixup class bases
80 		root.visit!ClassDeclaration((decl){
81 			if( decl.baseClass && decl.baseClass.typeDecl && !cast(ClassDeclaration)decl.baseClass.typeDecl )
82 				decl.baseClass = null;
83 			foreach( i; decl.derivedInterfaces )
84 				if( i.typeDecl && !cast(InterfaceDeclaration)i.typeDecl )
85 					i.typeDecl = null;
86 			assert(decl);
87 		});
88 
89 		// fixup interface bases
90 		root.visit!InterfaceDeclaration((decl){
91 			foreach( i; decl.derivedInterfaces )
92 				if( i.typeDecl && !cast(InterfaceDeclaration)i.typeDecl )
93 					i.typeDecl = null;
94 			assert(decl);
95 		});
96 	}
97 
98 	void parseModule(Json json, Package root_package)
99 	{
100 		Module mod;
101 		if( "name" !in json ){
102 			logError("No name attribute in module %s - ignoring", json.filename.opt!string);
103 			return;
104 		}
105 		auto path = json.name.get!string.split(".");
106 		Package p = root_package;
107 		foreach( i, pe; path ){
108 			if( i+1 < path.length ) p = p.getOrAddPackage(pe);
109 			else mod = p.createModule(pe);
110 		}
111 
112 		mod.file = json.file.get!string;
113 		mod.docGroup = new DocGroup(mod, json.comment.opt!string());
114 		mod.members = parseDeclList(json.members, mod);
115 	}
116 
117 	Declaration[] parseDeclList(Json json, Entity parent)
118 	{
119 		if( json.type == Json.Type.Undefined ) return null;
120 		DocGroup lastdoc;
121 		Declaration[] ret;
122 		foreach( mem; json ){
123 			auto decl = parseDecl(mem, parent);
124 			if( !decl ) continue;
125 			auto doc = decl.docGroup;
126 			if( lastdoc && (doc.text == lastdoc.text && doc.text.length || doc.comment.isDitto) ){
127 				lastdoc.members ~= decl;
128 				decl.docGroup = lastdoc;
129 			} else if( doc.comment.isPrivate ){
130 				decl.protection = Protection.Private;
131 				lastdoc = null;
132 			} else lastdoc = decl.docGroup;
133 			ret ~= decl;
134 		}
135 		return ret;
136 	}
137 
138 	Declaration parseDecl(Json json, Entity parent)
139 	{
140 		Declaration ret;
141 
142 		// DMD outputs templates with the wrong kind sometimes
143 		if (json.name.get!string().canFind('(') && json.kind != "mixin") {
144 			ret = parseTemplateDecl(json, parent);
145 		} else {
146 			switch( json.kind.get!string ){
147 				default:
148 					logWarn("Unknown declaration kind: %s", json.kind.get!string);
149 					return null;
150 				case "import":
151 				case "static import":
152 					// TODO: use for symbol resolving
153 					return null;
154 				case "destructor": return null;
155 				case "mixin": return null;
156 				case "alias":
157 					ret = parseAliasDecl(json, parent);
158 					break;
159 				case "function":
160 				case "allocator":
161 				case "deallocator":
162 				case "constructor":
163 					ret = parseFunctionDecl(json, parent);
164 					break;
165 				case "enum":
166 					ret = parseEnumDecl(json, parent);
167 					break;
168 				case "enum member":
169 					ret = parseEnumMemberDecl(json, parent);
170 					break;
171 				case "struct":
172 				case "union":
173 				case "class":
174 				case "interface":
175 					ret = parseCompositeDecl(json, parent);
176 					break;
177 				case "variable":
178 					ret = parseVariableDecl(json, parent);
179 					break;
180 				case "template":
181 					ret = parseTemplateDecl(json, parent);
182 					break;
183 			}
184 		}
185 
186 		ret.protection = parseProtection(json.protection);
187 		ret.line = json["line"].opt!int;
188 		ret.docGroup = new DocGroup(ret, json.comment.opt!string());
189 
190 		return ret;
191 	}
192 
193 	auto parseAliasDecl(Json json, Entity parent)
194 	{
195 		auto ret = new AliasDeclaration(parent, json.name.get!string);
196 		ret.attributes = json.storageClass.opt!(Json[]).map!(j => j.get!string).array;
197 		ret.targetType = parseType(json, ret, null);
198 		if( ret.targetType && ret.targetType.kind == TypeKind.Primitive && ret.targetType.typeName.length == 0 )
199 			ret.targetType = null;
200 		insertIntoTypeMap(ret);
201 		return ret;
202 	}
203 
204 	auto parseFunctionDecl(Json json, Entity parent)
205 	{
206 		auto ret = new FunctionDeclaration(parent, json.name.opt!string);
207 		ret.type = parseType(json, ret, "void()");
208 		// TODO: use "storageClass" and "parameters" fields
209 		if( ret.type.kind == TypeKind.Function ){
210 			ret.returnType = ret.type.returnType;
211 			ret.attributes = ret.type.attributes ~ ret.type.modifiers;
212 			if (auto psc = "storageClass" in json)
213 				foreach (sc; *psc)
214 					if (!ret.attributes.canFind(sc.get!string))
215 						ret.attributes ~= sc.get!string;
216 
217 			auto params = json.parameters.opt!(Json[]);
218 			if (!params) {
219 				params.length = ret.type.parameterTypes.length;
220 				foreach (i, pt; ret.type.parameterTypes) {
221 					auto jp = Json.emptyObject;
222 					jp.name = ret.type._parameterNames[i];
223 					jp["type"] = pt.text;
224 					if (ret.type._parameterDefaultValues[i])
225 						jp["default"] = ret.type._parameterDefaultValues[i].valueString;
226 					params[i] = jp;
227 				}
228 			}
229 
230 			foreach (i, p; params) {
231 				auto pname = p.name.opt!string;
232 				auto decl = new VariableDeclaration(ret, pname);
233 				foreach (sc; p["storageClass"].opt!(Json[]))
234 					if (!decl.attributes.canFind(sc.get!string))
235 						decl.attributes ~= sc.get!string;
236 				decl.type = parseType(p, ret);
237 				if (auto pdv = "default" in p)
238 					decl.initializer = parseValue(pdv.opt!string);
239 				ret.parameters ~= decl;
240 			}
241 		} else {
242 			logError("Expected function type for '%s'/'%s', got %s %s", json["type"].opt!string, demangleType(json["deco"].opt!string), ret.type.kind, ret.type.typeName);
243 		}
244 		return ret;
245 	}
246 
247 	auto parseEnumDecl(Json json, Entity parent)
248 	{
249 		auto ret = new EnumDeclaration(parent, json.name.get!string);
250 		insertIntoTypeMap(ret);
251 		if( "base" !in json ){ // FIXME: parse deco instead
252 			if( auto pd = "baseDeco" in json )
253 				json.base = demanglePrettyType(pd.get!string());
254 		}
255 		ret.baseType = parseType(json.base, ret);
256 		auto mems = parseDeclList(json.members, ret);
257 		foreach( m; mems ){
258 			auto em = cast(EnumMemberDeclaration)m;
259 			assert(em !is null, "Enum containing non-enum-members?");
260 			ret.members ~= em;
261 		}
262 		return ret;
263 	}
264 
265 	auto parseEnumMemberDecl(Json json, Entity parent)
266 	{
267 		auto ret = new EnumMemberDeclaration(parent, json.name.get!string);
268 		if (json.value.opt!string.length)
269 			ret.value = parseValue(json.value.opt!string);
270 		return ret;
271 	}
272 
273 	auto parseCompositeDecl(Json json, Entity parent)
274 	{
275 		CompositeTypeDeclaration ret;
276 		switch(json.kind.get!string){
277 			default:
278 				logWarn("Invalid composite decl kind: %s", json.kind.get!string);
279 				return new StructDeclaration(parent, json.name.get!string);
280 			case "struct":
281 				ret = new StructDeclaration(parent, json.name.get!string);
282 				break;
283 			case "union":
284 				ret = new UnionDeclaration(parent, json.name.get!string);
285 				break;
286 			case "class":
287 				auto clsdecl = new ClassDeclaration(parent, json.name.get!string);
288 				if( clsdecl.qualifiedName != "object.Object" )
289 					clsdecl.baseClass = parseType(json.base, clsdecl, "Object", false);
290 				foreach( intf; json.interfaces.opt!(Json[]) )
291 					clsdecl.derivedInterfaces ~= parseType(intf, clsdecl);
292 				ret = clsdecl;
293 				break;
294 			case "interface":
295 				auto intfdecl = new InterfaceDeclaration(parent, json.name.get!string);
296 				foreach( intf; json.interfaces.opt!(Json[]) )
297 					intfdecl.derivedInterfaces ~= parseType(intf, intfdecl);
298 				ret = intfdecl;
299 				break;
300 		}
301 
302 		insertIntoTypeMap(ret);
303 
304 		ret.members = parseDeclList(json.members, ret);
305 
306 		return ret;
307 	}
308 
309 	Declaration parseVariableDecl(Json json, Entity parent)
310 	{
311 		if (json.storageClass.opt!(Json[]).canFind!(j => j.opt!string == "enum")) {
312 			auto ret = new EnumMemberDeclaration(parent, json.name.get!string);
313 			if (json["init"].opt!string.length)
314 				ret.value = parseValue(json["init"].opt!string);
315 			return ret;
316 		} else {
317 			auto ret = new VariableDeclaration(parent, json.name.get!string);
318 			ret.type = parseType(json, ret);
319 			if (json["init"].opt!string.length)
320 				ret.initializer = parseValue(json["init"].opt!string);
321 			return ret;
322 		}
323 	}
324 
325 	auto parseTemplateDecl(Json json, Entity parent)
326 	{
327 		auto ret = new TemplateDeclaration(parent, json.name.get!string);
328 		foreach (arg; json.parameters.opt!(Json[])) {
329 			string argstr;
330 			switch (arg.kind.get!string) {
331 				case "value":
332 					if (auto pt = "type" in arg) argstr = pt.get!string ~ ' ';
333 					else argstr = demanglePrettyType(arg.deco.get!string) ~ ' ';
334 					goto default;
335 				case "alias":
336 					argstr = "alias ";
337 					goto default;
338 				case "tuple":
339 					argstr ~= arg.name.get!string ~ "...";
340 					break;
341 				default:
342 					argstr ~= arg.name.get!string;
343 			}
344 			ret.templateArgs ~= new TemplateParameterDeclaration(ret, argstr);
345 			ret.templateConstraint = json.constraint.opt!string;
346 		}
347 		ret.members = parseDeclList(json.members, ret);
348 		return ret;
349 	}
350 
351 	Type parseType(Json json, Entity sc, string def_type = "void", bool warn_id_not_exists = true)
352 	{
353 		string str;
354 		if( json.type == Json.Type.Undefined ){
355 			if (warn_id_not_exists) logWarn("No type found for %s.", sc.qualifiedName);
356 			str = def_type;
357 		} else if( json.type == Json.Type.String ) str = json.get!string();
358 		else if( auto pv = "deco" in json ) str = demanglePrettyType(pv.get!string());
359 		else if( auto pv = "type" in json ) str = pv.get!string();
360 		else if( auto pv = "originalType" in json ) str = pv.get!string();
361 
362 		if( str.length == 0 ) str = def_type;
363 
364 		if( !str.length ) return null;
365 
366 		auto tokens = tokenizeDSource(str);
367 		
368 		logDebug("parse type '%s'", str);
369 		try {
370 			auto type = parseTypeDecl(tokens, sc);
371 			type.text = str;
372 			return type;
373 		} catch( Exception e ){
374 			logError("Error parsing type '%s': %s", str, e.msg);
375 			auto type = new Type;
376 			type.text = str;
377 			type.typeName = str;
378 			type.kind = TypeKind.Primitive;
379 			return type;
380 		}
381 	}
382 
383 	Value parseValue(string str)
384 	{
385 		auto ret = new Value;
386 		//ret.type = ;
387 		ret.valueString = str;
388 		return ret;
389 	}
390 
391 	Protection parseProtection(Json prot)
392 	{
393 		switch( prot.opt!string ){
394 			default: return Protection.Public;
395 			case "package": return Protection.Package;
396 			case "protected": return Protection.Protected;
397 			case "private": return Protection.Private;
398 		}
399 	}
400 
401 	Declaration lookupDecl(string qualified_name, Entity sc)
402 	{
403 		while(sc){
404 			auto ent = cast(Declaration)sc.lookup(qualified_name);
405 			if( ent ) return ent;
406 			sc = sc.parent;
407 		}
408 		return null;
409 	}
410 
411 	Type parseTypeDecl(ref string[] tokens, Entity sc)
412 	{
413 
414 		auto ret = parseType(tokens, sc);
415 		return ret;
416 	}
417 
418 	Type parseType(ref string[] tokens, Entity sc)
419 	{
420 		string[] attributes;
421 		auto basic_type = parseBasicType(tokens, sc, attributes);
422 		basic_type.attributes ~= attributes;
423 		return basic_type;	
424 	}
425 
426 	Type parseBasicType(ref string[] tokens, Entity sc, out string[] attributes)
427 	{
428 		static immutable global_attribute_keywords = ["abstract", "auto", "const", "deprecated", "enum",
429 			"extern", "final", "immutable", "inout", "shared", "nothrow", "override", "pure",
430 			"__gshared", "scope", "static", "synchronize"];
431 
432 		static immutable parameter_attribute_keywords = ["auto", "const", "final", "immutable", "in", "inout",
433 			"lazy", "out", "ref", "scope", "shared"];
434 
435 		static immutable member_function_attribute_keywords = ["const", "immutable", "inout", "shared", "pure", "nothrow"];
436 		
437 			
438 		if( tokens.length > 0 && tokens[0] == "extern" ){
439 			enforce(tokens[1] == "(");
440 			enforce(tokens[3] == ")");
441 			attributes ~= join(tokens[0 .. 4]);
442 			tokens = tokens[4 .. $];
443 		}
444 		
445 		immutable string[] attribute_keywords = global_attribute_keywords ~ parameter_attribute_keywords ~ member_function_attribute_keywords;
446 		/*final switch( sc ){
447 			case DeclScope.Global: attribute_keywords = global_attribute_keywords; break;
448 			case DeclScope.Parameter: attribute_keywords = parameter_attribute_keywords; break;
449 			case DeclScope.Class: attribute_keywords = member_function_attribute_keywords; break;
450 		}*/
451 
452 		void parseAttributes(ref string[] dst, const(string)[] keywords)
453 		{
454 			while( tokens.length > 0 ){
455 				if( tokens.front == "@" ){
456 					tokens.popFront();
457 					dst ~= "@"~tokens.front;
458 					tokens.popFront();
459 				} else if( keywords.countUntil(tokens[0]) >= 0 && tokens[1] != "(" ){
460 					dst ~= tokens.front;
461 					tokens.popFront();
462 				} else break;
463 			}
464 		}
465 
466 		parseAttributes(attributes, attribute_keywords);
467 
468 
469 		Type type;
470 		static immutable const_modifiers = ["const", "immutable", "shared", "inout"];
471 		if (tokens.length > 2 && tokens[1] == "(" && const_modifiers.countUntil(tokens[0]) >= 0) {
472 			auto mod = tokens.front;
473 			tokens.popFrontN(2);
474 			string[] subattrs;
475 			type = parseBasicType(tokens, sc, subattrs);
476 			type.modifiers ~= mod;
477 			type.attributes ~= subattrs;
478 			enforce(!tokens.empty && tokens.front == ")", format("Missing ')' for '%s('", mod));
479 			tokens.popFront();
480 		} else {
481 			type = new Type;
482 			type.kind = TypeKind.Primitive;
483 			m_primTypes ~= tuple(type, sc);
484 
485 			size_t start = 0, end;
486 			if( tokens[start] == "." ) start++;
487 			for( end = start; end < tokens.length && isIdent(tokens[end]); ){
488 				end++;
489 				if( end >= tokens.length || tokens[end] != "." )
490 					break;
491 				end++;
492 			}
493 
494 			size_t i = end;
495 
496 			string type_name, nested_name;
497 			if( i == 0 && tokens[0] == "..." ){
498 				type_name = "...";
499 				nested_name = null;
500 			} else if( i == 0 && tokens[0] == "(" ){
501 				type_name = "constructor";
502 				nested_name = null;
503 			} else {
504 				enforce(i > 0, "Expected identifier but got "~tokens.front);
505 				type.typeName = join(tokens[start .. end]);
506 				//type.typeDecl = cast(Declaration)sc.lookup(type.typeName);
507 				tokens.popFrontN(i);
508 				
509 				if (type.typeName == "typeof" && !tokens.empty && tokens.front == "(") {
510 					type.typeName ~= "(";
511 					tokens.popFront();
512 					int level = 1;
513 					while (!tokens.empty && level > 0) {
514 						if (tokens.front == "(") level++;
515 						else if( tokens.front == ")") level--;
516 						type.typeName ~= tokens.front;
517 						tokens.popFront();
518 					}
519 				} else if( !tokens.empty && tokens.front == "!" ){
520 					tokens.popFront();
521 					if( tokens.front == "(" ){
522 						size_t j = 1;
523 						int cc = 1;
524 						while( cc > 0 ){
525 							assert(j < tokens.length);
526 							if( tokens[j] == "(" ) cc++;
527 							else if( tokens[j] == ")") cc--;
528 							j++;
529 						}
530 						type.templateArgs = join(tokens[0 .. j]);
531 						tokens.popFrontN(j);
532 						logDebug("templargs: %s", type.templateArgs);
533 					} else {
534 						type.templateArgs = tokens[0];
535 						tokens.popFront();
536 					}
537 					
538 					// HACK: dropping the actual type name here!
539 					while (!tokens.empty && tokens.front == ".") {
540 						tokens.popFront();
541 						if (!tokens.empty()) tokens.popFront();
542 					}
543 				}
544 			}
545 		}
546 		
547 		while( !tokens.empty ){
548 			if( tokens.front == "*" ){
549 				auto ptr = new Type;
550 				ptr.kind = TypeKind.Pointer;
551 				ptr.elementType = type;
552 				type = ptr;
553 				tokens.popFront();
554 			} else if( tokens.front == "[" ){
555 				tokens.popFront();
556 				if( tokens.front == "]" ){
557 					auto arr = new Type;
558 					arr.kind = TypeKind.Array;
559 					arr.elementType = type;
560 					type = arr;
561 				} else {
562 					string[] tokens_copy = tokens;
563 					Type keytp;
564 					if (!isDigit(tokens.front[0]) && tokens.front != "!") keytp = parseType(tokens_copy, sc);
565 					if (keytp && !tokens_copy.empty && tokens_copy.front == "]") {
566 						tokens = tokens_copy;
567 						logDebug("GOT TYPE: %s", keytp.toString());
568 						auto aa = new Type;
569 						aa.kind = TypeKind.AssociativeArray;
570 						aa.elementType = type;
571 						aa.keyType = keytp;
572 						type = aa;
573 					} else {
574 						auto arr = new Type;
575 						arr.kind = TypeKind.StaticArray;
576 						arr.elementType = type;
577 						arr.arrayLength = tokens.front;
578 						tokens.popFront();
579 						while (!tokens.empty && tokens.front != "]") {
580 							arr.arrayLength ~= tokens.front;
581 							tokens.popFront();
582 						}
583 						type = arr;
584 					}
585 				}
586 				enforce(tokens.front == "]", "Expected ']', got '"~tokens.front~"'.");
587 				tokens.popFront();
588 			} else break;
589 		}
590 		
591 		while (!tokens.empty && (tokens.front == "function" || tokens.front == "delegate" || tokens.front == "(")) {
592 			Type ftype = new Type;
593 			ftype.kind = tokens.front == "(" || tokens.front == "function" ? TypeKind.Function : TypeKind.Delegate;
594 			ftype.returnType = type;
595 			if (tokens.front != "(") tokens.popFront();
596 			enforce(tokens.front == "(");
597 			tokens.popFront();
598 			if (!tokens.empty && tokens.front == ",") tokens.popFront(); // sometimes demangleType() returns something like "void(, ...)"
599 			while (true) {
600 				if (tokens.front == ")") break;
601 				enforce(!tokens.empty);
602 				ftype.parameterTypes ~= parseTypeDecl(tokens, sc);
603 				if (tokens.front != "," && tokens.front != ")") {
604 					ftype._parameterNames ~= tokens.front;
605 					tokens.popFront();
606 				} else ftype._parameterNames ~= null;
607 				if (tokens.front == "...") {
608 					ftype._parameterNames[$-1] ~= tokens.front;
609 					tokens.popFront();
610 				}
611 				if (tokens.front == "=") {
612 					tokens.popFront();
613 					string defval;
614 					int ccount = 0;
615 					while (!tokens.empty) {
616 						if (ccount == 0 && (tokens.front == "," || tokens.front == ")"))
617 							break;
618 						if (tokens.front == "(") ccount++;
619 						else if (tokens.front == ")") ccount--;
620 						defval ~= tokens.front;
621 						tokens.popFront();
622 					}
623 					ftype._parameterDefaultValues ~= parseValue(defval);
624 					logDebug("got defval %s", defval);
625 				} else ftype._parameterDefaultValues ~= null;
626 				if (tokens.front == ")") break;
627 				enforce(tokens.front == ",", "Expecting ',', got "~tokens.front);
628 				tokens.popFront();
629 			}
630 			tokens.popFront();
631 
632 			parseAttributes(ftype.attributes, member_function_attribute_keywords);
633 
634 			type = ftype;
635 		}
636 
637 		return type;
638 	}
639 	
640 	string[] tokenizeDSource(string dsource_)
641 	{
642 		static immutable dstring[] tokens = [
643 			"/", "/=", ".", "..", "...", "&", "&=", "&&", "|", "|=", "||",
644 			"-", "-=", "--", "+", "+=", "++", "<", "<=", "<<", "<<=",
645 			"<>", "<>=", ">", ">=", ">>=", ">>>=", ">>", ">>>", "!", "!=",
646 			"!<>", "!<>=", "!<", "!<=", "!>", "!>=", "(", ")", "[", "]",
647 			"{", "}", "?", ",", ";", ":", "$", "=", "==", "*", "*=",
648 			"%", "%=", "^", "^=", "~", "~=", "@", "=>", "#"
649 		];
650 		static bool[dstring] token_map;
651 		
652 		if( !token_map.length ){
653 			foreach( t; tokens )
654 				token_map[t] = true;
655 			token_map.rehash;
656 		}
657 		
658 		dstring dsource = to!dstring(dsource_);
659 		
660 		dstring[] ret;
661 		outer:
662 		while(true){
663 			dsource = stripLeft(dsource);
664 			if( dsource.length == 0 ) break;
665 			
666 			// special token?
667 			foreach_reverse( i; 1 .. min(5, dsource.length+1) )
668 				if( dsource[0 .. i] in token_map ){
669 					ret ~= dsource[0 .. i];
670 					dsource.popFrontN(i);
671 					continue outer;
672 				}
673 			
674 			// identifier?
675 			if( dsource[0] == '_' || std.uni.isAlpha(dsource[0]) ){
676 				size_t i = 1;
677 				while( i < dsource.length && (dsource[i] == '_' || std.uni.isAlpha(dsource[i]) || isDigit(dsource[i])) ) i++;
678 				ret ~= dsource[0 .. i];
679 				dsource.popFrontN(i);
680 				continue;
681 			}
682 			
683 			// character literal?
684 			if( dsource[0] == '\'' ){
685 				size_t i = 1;
686 				while( dsource[i] != '\'' ){
687 					if( dsource[i] == '\\' ) i++;
688 					i++;
689 					enforce(i < dsource.length);
690 				}
691 				ret ~= dsource[0 .. i+1];
692 				dsource.popFrontN(i+1);
693 				continue;
694 			}
695 			
696 			// string? (incomplete!)
697 			if( dsource[0] == '"' ){
698 				size_t i = 1;
699 				while( dsource[i] != '"' ){
700 					if( dsource[i] == '\\' ) i++;
701 					i++;
702 					enforce(i < dsource.length);
703 				}
704 				ret ~= dsource[0 .. i+1];
705 				dsource.popFrontN(i+1);
706 				continue;
707 			}
708 			
709 			// number?
710 			if( isDigit(dsource[0]) || dsource[0] == '.' ){
711 				auto dscopy = dsource;
712 				parse!double(dscopy);
713 				ret ~= dsource[0 .. dsource.length-dscopy.length];
714 				dsource.popFrontN(dsource.length-dscopy.length);
715 				if( dsource.startsWith("u") ) dsource.popFront();
716 				else if( dsource.startsWith("f") ) dsource.popFront();
717 				continue;
718 			}
719 			
720 			ret ~= dsource[0 .. 1];
721 			dsource.popFront();
722 		}
723 		
724 		auto ret_ = new string[ret.length];
725 		foreach( i; 0 .. ret.length ) ret_[i] = to!string(ret[i]);
726 		return ret_;
727 	}
728 
729 	bool isDigit(dchar ch)
730 	{
731 		return ch >= '0' && ch <= '9';
732 	}
733 
734 	bool isIdent(string str)
735 	{
736 		if( str.length < 1 ) return false;
737 		foreach( i, dchar ch; str ){
738 			if( ch == '_' || std.uni.isAlpha(ch) ) continue;
739 			if( i > 0 && isDigit(ch) ) continue;
740 			return false;
741 		}
742 		return true;	
743 	}
744 
745 	string fullStrip(string s)
746 	{
747 		string chars = " \t\r\n";
748 		while( s.length > 0 && chars.countUntil(s[0]) >= 0 ) s.popFront();
749 		while( s.length > 0 && chars.countUntil(s[$-1]) >= 0 ) s.popBack();
750 		return s;
751 	}
752 
753 	void insertIntoTypeMap(Declaration decl)
754 	{
755 		string[] parts = split(decl.qualifiedName, ".");
756 		foreach( i; 0 .. parts.length ){
757 			auto partial_name = join(parts[i .. $], ".");
758 			m_typeMap[partial_name] = decl;
759 		}
760 	}
761 }
762 
763 string demanglePrettyType(string mangled_type)
764 {
765 	if (mangled_type == "n") return "typeof(null)"; // Workaround D issue 14410
766 	auto str = assumeUnique(demangleType(mangled_type));
767 	str = str.replace("immutable(char)[]", "string");
768 	str = str.replace("immutable(wchar)[]", "wstring");
769 	str = str.replace("immutable(dchar)[]", "dstring");
770 	return str;
771 }