[WIP] Hierarchy generation fully works now

All the types, including enums, are properly included and nested in their
namespaces and classes.

Issue: enums don't have their full managed names generated properly yet
Issue: type index can't handle conflicts properly yet (e.g. we have two
namespaces with different full native name but the same full managed name -
Android.App. This is a legitimate scenario, need to come up with a way to handle
it)

Author: grendello

api.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<api api-source="class-parse" platform="10">
+<api api-source="class-parse" platform="27">
   <package name="android" jni-name="android">
     <class abstract="false" deprecated="not deprecated" jni-extends="Ljava/lang/Object;" extends="java.lang.Object" extends-generic-aware="java.lang.Object" final="true" name="Manifest" jni-signature="Landroid/Manifest;" static="false" visibility="public">
       <constructor deprecated="not deprecated" final="false" name="Manifest" static="false" visibility="public" bridge="false" synthetic="false" jni-signature="()V" />

metadata

@@ -314,7 +314,7 @@
   <attr path="/api/package[@name='java.nio.channels']/class[@name='Channels']/method[@name='newChannel' and @return='java.nio.channels.WritableByteChannel']" name="managedName">NewWritableChannel</attr>
   <attr path="/api/package[@name='java.util']/class[@name='Dictionary']/method/parameter[@name='key']" name="type">K</attr>
   <attr path="/api/package[@name='java.util']/class[@name='Dictionary']/method/parameter[@name='value']" name="type">V</attr>
-  <attr path="/api/package[@name='java.util']/class[@name='Dictionary']" name="name">Dictionary&lt;K, V&gt;</attr>
+  <attr path="/api/package[@name='java.util']/class[@name='Dictionary']" name="name-generic-aware">Dictionary&lt;K, V&gt;</attr>
   <attr path="/api/package[@name='java.util']/class[@name='Hashtable']/method/parameter[@name='key']" name="type">K</attr>
   <attr path="/api/package[@name='java.util']/class[@name='Hashtable']/method/parameter[@name='value']" name="type">V</attr>
   <attr path="/api/package[@name='java.util']/class[@name='Hashtable']/implements[@name='java.util.Map']" name="name-generic-aware">java.util.Map&lt;K, V&gt;</attr>

src/Java.Interop.Bindings/Compiler/DefaultOutputPathProvider.cs

@@ -55,6 +55,10 @@ public override FilesystemPath GetPathFor (string rootDirectory, HierarchyElemen
 					relativePath = GetFilePath (iface);
 					break;
 
+				case HierarchyEnum enm:
+					relativePath = GetFilePath (enm);
+					break;
+
 				default:
 					throw new InvalidOperationException ($"Unsupported hierarchy type '{element.GetType ()}'");
 			}

src/Java.Interop.Bindings/Compiler/Generator.cs

@@ -107,7 +107,7 @@ protected virtual void GenerateNamespace (string outputDirectoryRoot, HierarchyN
 			}
 		}
 
-		protected virtual void GenerateNamespaceMember (HierarchyElement element, StreamWriter writer, string outputFileName)
+		protected virtual void GenerateNamespaceMember (HierarchyElement element, TextWriter writer, string outputFileName)
 		{
 			Logger.Debug ($"Generating {element.GetManagedName (true)} in namespace output file: {outputFileName}");
 			OutputNamespaceMember (element, writer, outputFileName);
@@ -117,7 +117,7 @@ protected virtual void GenerateNamespaceMember (HierarchyNamespace ns, Hierarchy
 		{
 			FilesystemPath path = Context.OutputPathProvider.GetPathFor (outputDirectory, element);
 			if (String.IsNullOrEmpty (path?.FullPath)) {
-				Logger.Warning ($"Unable to generate output for element {element.GetManagedName (true)} since no full path was given");
+				Logger.Warning ($"Unable to generate output for element {element.GetType ().FullName} ({element.GetManagedName (true) ?? element.FullName}) since no full path was given (at {element.GetLocation ()})");
 				return;
 			}
 
@@ -136,15 +136,15 @@ protected virtual void GenerateNamespaceMember (HierarchyNamespace ns, Hierarchy
 			}
 		}
 
-		protected virtual void OutputNamespaceMember (HierarchyElement element, StreamWriter writer, string fileName)
+		protected virtual void OutputNamespaceMember (HierarchyElement element, TextWriter writer, string fileName)
 		{
 		}
 
-		protected virtual void WriteFileHeader (HierarchyNamespace ns, StreamWriter writer)
+		protected virtual void WriteFileHeader (HierarchyNamespace ns, TextWriter writer)
 		{
 		}
 
-		protected virtual void WriteFileFooter (HierarchyNamespace ns, StreamWriter writer)
+		protected virtual void WriteFileFooter (HierarchyNamespace ns, TextWriter writer)
 		{
 		}
 

src/Java.Interop.Bindings/Compiler/Hierarchy.cs

@@ -44,14 +44,12 @@ public class Hierarchy : HierarchyBase
 		public const string DefaultInterfaceBaseType = DefaultInterfaceBaseTypeNamespace + ".IJavaObject";
 		public const string DefaultClassBaseType = DefaultClassBaseTypeNamespace + ".Object";
 
-		Dictionary<string, HierarchyObject> typeIndex;
 		List<HierarchyNamespace> namespaces;
 		List<HierarchyEnum> enums;
 		Dictionary<string, List<HierarchyElement>> typeNameDependants;
 
 		public IList<HierarchyNamespace> Namespaces => namespaces;
-		public IList<HierarchyEnum> Enums => enums;
-		protected Dictionary<string, HierarchyObject> TypeIndex => typeIndex;
+		protected HierarchyIndex TypeIndex { get; } = new HierarchyIndex ();
 
 		public void Build (IList<ApiElement> rawElements)
 		{ 
@@ -92,69 +90,96 @@ public void Build (IList<ApiElement> rawElements)
 			// interfaces that don't derive from other interfaces
 			HierarchyInterface iJavaLangObject = androidRuntime.Members?.OfType <HierarchyInterface> ().Where (iface => String.Compare (DefaultInterfaceBaseType, iface?.FullName, StringComparison.OrdinalIgnoreCase) == 0).FirstOrDefault ();
 			if (iJavaLangObject == null) {
-				Logger.Verbose ("Synthetizing Android.Runtime.IJavaObject interface (not found after parsing API description)");
+				Logger.Verbose ("Synthesizing Android.Runtime.IJavaObject interface (not found after parsing API description)");
 				iJavaLangObject = CreateHierarchyElementInternal <HierarchyInterface> (androidRuntime);
 				iJavaLangObject.Init ();
 				iJavaLangObject.FullName = DefaultInterfaceBaseType;
 				iJavaLangObject.Name = Helpers.GetBaseTypeName (iJavaLangObject.FullName);
 				iJavaLangObject.IgnoreForCodeGeneration = true;
 				iJavaLangObject.InvokerNotNeeded = true;
 				iJavaLangObject.UseGlobal = true;
-				AddTypeToIndex (iJavaLangObject);
+				TypeIndex.Add (iJavaLangObject);
 			}
 
-			// Pass 2: nest classes, enums and interfaces
-			NestElements ();
+			// Pass 2: nest classes, interfaces
+			NestNamespaces ();
 
-			// Pass 3: generate and inject synthetic elements
+			// Pass 3: generate managed names
+			Helpers.ForEachNotNull (namespaces, (HierarchyNamespace ns) => GenerateManagedNames (ns));
+
+			// TODO: fix generation of full managed names for enums
+			Helpers.ForEachNotNull (enums, (HierarchyEnum enm) => GenerateManagedNames (enm));
+
+			// Pass 4: nest enums (they need managed names)
+			NestEnums ();
+
+			// Pass 5: generate and inject synthetic elements
 			SynthesizeElements ();
 
-			// Pass 4: resolve base types since we have everything in place now
+			// Pass 6: resolve base types since we have everything in place now
 			ResolveBaseTypes ();
 
-			// Pass 5: generate managed names and rename all the related nested types accordingly
+			// Pass 7: sort class members
+		}
+
+		protected void GenerateManagedNames (HierarchyElement root)
+		{
+			if (root == null)
+				return;
+
+			root.SetManagedNames ();
+			if (root is HierarchyObject || root is HierarchyNamespace)
+			 	TypeIndex.AddManaged (root);
+			if (!root.HasMembers)
+				return;
 
-			// Pass 6: sort class members
+			Helpers.ForEachNotNull(root.Members, (HierarchyElement element) => GenerateManagedNames (element));
 		}
 
 		protected void ResolveBaseTypes ()
 		{
-			if (namespaces == null || namespaces.Count == 0 || typeIndex == null)
+			if (namespaces == null || namespaces.Count == 0)
 				return;
 
 			foreach (HierarchyNamespace ns in namespaces) {
 				if (ns == null || ns.Members == null || ns.Members.Count == 0)
 					continue;
 
 				foreach (HierarchyObject type in ns.Members.OfType<HierarchyObject> ().Where (o => o != null)) {
-					type.ResolveBaseTypes (typeIndex);
+					type.ResolveBaseTypes (TypeIndex);
 				}
 			}
 		}
 
 		protected virtual void SynthesizeElements ()
 		{}
 
-		protected void NestElements ()
+		void NestElements <TElement, TParent> (IList<TElement> elements, Dictionary <HierarchyElement, HierarchyElement> toReparent)
+			where TElement: HierarchyElement
+			where TParent: HierarchyElement
 		{
-			if (namespaces == null || namespaces.Count == 0)
+			if (elements == null)
 				return;
 
-			var toReparent = new Dictionary <HierarchyElement, HierarchyElement> ();
-			foreach (HierarchyNamespace ns in namespaces) {
-				if (ns == null || ns.Members == null || ns.Members.Count == 0)
+			foreach (HierarchyElement element in elements) {
+				TParent newParent = SelectNewParent <TParent> (element);
+				if (newParent == null)
 					continue;
 
-				foreach (HierarchyElement element in ns.Members) {
-					HierarchyClass newParent = SelectNewParent (element);
-					if (newParent == null)
-						continue;
+				if (toReparent.ContainsKey (element))
+					Logger.Warning ($"Element {element.Name} ({element.GetType ()} was already re-parented");
 
-					if (toReparent.ContainsKey (element))
-						Logger.Warning ($"Element {element.Name} ({element.GetType ()} was already re-parented");
-					toReparent [element] = newParent;
-				}
+				toReparent [element] = newParent;
 			}
+		}
+
+		protected void NestElements (Action<Dictionary <HierarchyElement, HierarchyElement>> looper)
+		{
+			if (looper == null)
+				throw new ArgumentNullException (nameof (looper));
+
+			var toReparent = new Dictionary <HierarchyElement, HierarchyElement> ();
+			looper (toReparent);
 
 			if (toReparent.Count == 0)
 				return;
@@ -168,12 +193,31 @@ protected void NestElements ()
 			}
 		}
 
-		protected virtual HierarchyClass SelectNewParent (HierarchyElement element)
+		protected void NestNamespaces ()
+		{
+			NestElements (
+				toReparent => {
+					foreach (HierarchyNamespace ns in namespaces) {
+						NestElements <HierarchyElement, HierarchyClass> (ns?.Members, toReparent);
+					}
+				}
+			);
+		}
+
+		protected void NestEnums ()
+		{
+			NestElements (toReparent => NestElements <HierarchyEnum, HierarchyNamespace> (enums, toReparent));
+		}
+
+		protected virtual T SelectNewParent <T> (HierarchyElement element) where T: HierarchyElement
 		{
-			if (element == null || String.IsNullOrEmpty (element.FullName))
+			if (element == null)
 				return null;
 
-			if (typeIndex == null || typeIndex.Count == 0) {
+			if (String.IsNullOrEmpty (element.FullName))
+				throw new InvalidOperationException ("Each element must have a full name");
+
+			if (TypeIndex.Count == 0) {
 				Logger.Warning ($"Unable to select new parent for element {element.Name}, type index does not exist");
 				return null;
 			}
@@ -185,15 +229,20 @@ protected virtual HierarchyClass SelectNewParent (HierarchyElement element)
 			if (String.IsNullOrEmpty (nsOrTypeName))
 				return null;
 
-			HierarchyObject maybeParent;
-			if (!typeIndex.TryGetValue (nsOrTypeName, out maybeParent))
-				return null;
+			try {
+				HierarchyElement maybeParent = TypeIndex.Lookup (nsOrTypeName);
+				if (maybeParent == null)
+					return null;
 
-			var klass = maybeParent as HierarchyClass;
-			if (klass == null)
-				return null;
+				var ret = maybeParent as T;
+				if (ret == null)
+					return null;
 
-			return klass;
+				return ret;
+			} catch {
+				Logger.Fatal ($"Error selecting new parent for element '{element.FullName}' ({element.GetLocation ()})");
+				throw;
+			}
 		}
 
 		public void Dump (string outputFile)
@@ -204,7 +253,7 @@ public void Dump (string outputFile)
 			string indent = String.Empty;
 			using (var sw = new StreamWriter (outputFile, false, Encoding.UTF8)) {
 				foreach (HierarchyNamespace ns in namespaces) {
-					sw.WriteLine ($"Namespace: {ns.FullName}");
+					sw.WriteLine ($"Namespace: [native: {ns.Name} ({ns.FullName})] [managed: {ns.ManagedName} ({ns.FullManagedName})]");
 					Dump (sw, indent + "\t", ns.Members);
 				}
 			}
@@ -216,7 +265,7 @@ void Dump (StreamWriter sw, string indent, IList<HierarchyElement> members)
 				return;
 
 			foreach (HierarchyElement element in members) {
-				sw.WriteLine ($"{indent}{TypeToName (element)}: {element.FullName} [managed name: {element.ManagedName}; full managed name: {element.FullManagedName}]");
+				sw.WriteLine ($"{indent}{TypeToName (element)}: [native: {element.Name} ({element.FullName})] [managed: {element.ManagedName} ({element.FullManagedName})]");
 				Dump (sw, indent + "\t", element.Members);
 			}
 		}
@@ -280,6 +329,7 @@ protected virtual void Process (ApiNameSpace apiNameSpace)
 			});
 
 			Helpers.AddToList (ns, ref namespaces);
+			TypeIndex.Add (ns);
 		}
 
 		void AddLocationComment (ApiElement element, HierarchyElement hierarchyElement)
@@ -331,7 +381,7 @@ protected virtual void Process (HierarchyNamespace parent, ApiClass klass)
 			});
 
 			parent.AddMember (hierarchyClass);
-			AddTypeToIndex (hierarchyClass);
+			TypeIndex.Add (hierarchyClass);
 		}
 
 		protected virtual void Process (HierarchyObject parent, ApiTypeParameter typeParameter)
@@ -473,48 +523,15 @@ protected virtual void Process (HierarchyNamespace parent, ApiInterface iface)
 			});
 
 			parent.AddMember (hierarchyInterface);
-			AddTypeToIndex (hierarchyInterface);
+			TypeIndex.Add (hierarchyInterface);
 		}
 
 		protected virtual void Process (ApiEnum apiEnum)
 		{
 			var enm = CreateHierarchyElementInternal <HierarchyEnum> (this);
 			enm.Init (apiEnum);
 			Helpers.AddToList (enm, ref enums);
-		}
-
-		protected void AddTypeToIndex (HierarchyObject type)
-		{
-			if (type == null)
-				throw new ArgumentNullException (nameof (type));
-
-			// TODO: add aliasing for types that have NameGenericAware (AppImplements) and their generic
-			// name is different than the regular one - map the generic one to the type desribed by the
-			// non-generic name
-			AddTypeToIndex (type.FullName, type);
-			AddTypeToIndex (type.FullManagedName, type);
-			AddTypeToIndex (type.NameGenericAware, type);
-		}
-
-		void AddTypeToIndex (string typeName, HierarchyObject type)
-		{
-			if (String.IsNullOrEmpty (typeName))
-				return;
-
-			if (String.Compare ("java.lang.Comparable", typeName, StringComparison.OrdinalIgnoreCase) == 0) {
-				Logger.Debug ($"java.lang.Comparable being added: {type} ({type.FullName})");
-			}
-
-			if (typeIndex == null)
-				typeIndex = new Dictionary<string, HierarchyObject> (StringComparer.Ordinal);
-
-			HierarchyObject t;
-			if (typeIndex.TryGetValue (typeName, out t) && t != null) {
-				if (t.GetType () != type.GetType ())
-					throw new InvalidOperationException ($"Conflicting type index entry. Type '{typeName} ({type.GetType ().FullName})' must not replace '{t.FullName} ({t.GetType ().FullName})' since they are of different types");
-			}
-
-			typeIndex [typeName] = type;
+			TypeIndex.Add (enm);
 		}
 
 		protected T CreateHierarchyElementInternal <T> (HierarchyBase parent) where T: HierarchyElement
@@ -536,8 +553,6 @@ void AddTypeToIndex (string typeName, HierarchyObject type)
 				ret = new HierarchyNamespace (parent as Hierarchy);
 			else if (type == typeof (HierarchyClass))
 				ret = new HierarchyClass (parent as HierarchyNamespace);
-			else if (type == typeof (HierarchyTypeMember))
-				ret = new HierarchyTypeMember (parent as HierarchyObject);
 			else if (type == typeof (HierarchyImplements))
 				ret = new HierarchyImplements (parent as HierarchyObject);
 			else if (type == typeof (HierarchyMethod))

src/Java.Interop.Bindings/Compiler/HierarchyAttributeAnnotationGenerator.cs

@@ -33,7 +33,7 @@ public abstract class HierarchyAttributeAnnotationGenerator : HierarchyCustomAtt
 		protected HierarchyAttributeAnnotationGenerator (HierarchyAttributeAnnotation attribute) : base (attribute)
 		{}
 
-		protected override void OutputParameters (Stream output)
+		protected override void OutputParameters (TextWriter output)
 		{
 			OutputParameter (output, Attribute.JavaName);
 		}

src/Java.Interop.Bindings/Compiler/HierarchyAttributeIntDefinitionGenerator.cs

@@ -33,7 +33,7 @@ public abstract class HierarchyAttributeIntDefinitionGenerator : HierarchyCustom
 		protected HierarchyAttributeIntDefinitionGenerator (HierarchyAttributeIntDefinition attribute) : base (attribute)
 		{}
 
-		protected override void OutputParameters (Stream output)
+		protected override void OutputParameters (TextWriter output)
 		{
 			OutputParameter (output, Attribute.ConstantMember);
 			if (!String.IsNullOrEmpty (Attribute.JniField))

src/Java.Interop.Bindings/Compiler/HierarchyAttributeNamespaceMappingGenerator.cs

@@ -33,7 +33,7 @@ public abstract class HierarchyAttributeNamespaceMappingGenerator : HierarchyCus
 		protected HierarchyAttributeNamespaceMappingGenerator (HierarchyAttributeNamespaceMapping attribute) : base (attribute)
 		{}
 
-		protected override void OutputParameters (Stream output)
+		protected override void OutputParameters (TextWriter output)
 		{
 			bool haveJava = !String.IsNullOrEmpty (Attribute.Java);
 			bool haveManaged = !String.IsNullOrEmpty (Attribute.Managed);

src/Java.Interop.Bindings/Compiler/HierarchyAttributeObsoleteGenerator.cs

@@ -33,7 +33,7 @@ public abstract class HierarchyAttributeObsoleteGenerator : HierarchyCustomAttri
 		protected HierarchyAttributeObsoleteGenerator (HierarchyAttributeObsolete attribute) : base (attribute)
 		{}
 
-		protected override void OutputParameters (Stream output)
+		protected override void OutputParameters (TextWriter output)
 		{
 			if (!Attribute.IsError && String.IsNullOrEmpty (Attribute.Message))
 				return;