Add default convention to CallingConvention.

src/core/modules/memory/memory_function.cpp

@@ -35,6 +35,7 @@
 #include "memory_function.h"
 #include "memory_utilities.h"
 #include "memory_hooks.h"
+#include "memory_wrap.h"
 
 // DynamicHooks
 #include "conventions/x86MsCdecl.h"
@@ -149,17 +150,17 @@ CFunction::CFunction(unsigned long ulAddr, object oCallingConvention, object oAr
 	catch( ... )
 	{
 		PyErr_Clear();
-	
+
 		// A custom calling convention will be used...
 		m_eCallingConvention = CONV_CUSTOM;
-		object _oCallingConvention = oCallingConvention(m_tArgs, m_eReturnType);
+		m_oCallingConvention = oCallingConvention(m_tArgs, m_eReturnType);
 
 		// FIXME:
 		// This is required to fix a crash, but it will also cause a memory leak,
 		// because no calling convention object that is created via this method will ever be deleted.
 		// TODO: Pretty sure this was required due to the missing held type definition. It was added, but wasn't tested yet.
-		Py_INCREF(_oCallingConvention.ptr());
-		m_pCallingConvention = extract<ICallingConvention*>(_oCallingConvention);
+		Py_INCREF(m_oCallingConvention.ptr());
+		m_pCallingConvention = extract<ICallingConvention*>(m_oCallingConvention);
 
 		// We didn't allocate the calling convention, someone else is responsible for it.
 		m_bAllocatedCallingConvention = false;
@@ -188,6 +189,24 @@ CFunction::CFunction(unsigned long ulAddr, Convention_t eCallingConvention,
 
 CFunction::~CFunction()
 {
+	// If we created custom calling convention, clean it up.
+	// This does not apply to hooked calling convention.
+	if (!m_oCallingConvention.is_none())
+	{
+		CHook* pHook = GetHookManager()->FindHook((void *) m_ulAddr);
+		if (!pHook || pHook->m_pCallingConvention != m_pCallingConvention)
+		{
+			ICallingConventionWrapper* _pCallingConventionWrapper = extract<ICallingConventionWrapper*>(m_oCallingConvention);
+
+			Py_DECREF(m_oCallingConvention.ptr());
+
+			delete _pCallingConventionWrapper;
+			m_pCallingConvention = NULL;
+		}
+
+		return;
+	}
+
 	// If we didn't allocate the calling convention, then it is not our responsibility.
 	if (!m_bAllocatedCallingConvention)
 		return;

src/core/modules/memory/memory_function.h

@@ -102,6 +102,9 @@ class CFunction: public CPointer, private boost::noncopyable
 	// DynamicHooks calling convention (built-in and custom)
 	ICallingConvention*		m_pCallingConvention;
 	bool					m_bAllocatedCallingConvention;
+
+	// Custom calling convention
+	object					m_oCallingConvention = object();
 };
 
 

src/core/modules/memory/memory_wrap.cpp

@@ -825,13 +825,14 @@ void export_calling_convention(scope _memory)
 		"An an abstract class that is used to create custom calling "
 		"conventions (only available for hooking function and not for"
 		" calling functions).\n",
-		init< object, DataType_t, optional<int> >(
-			(arg("arg_types"), arg("return_type"), arg("alignment")),
+		init< object, DataType_t, optional<int, Convention_t> >(
+			(arg("arg_types"), arg("return_type"), arg("alignment")=4, arg("default_convention")=CONV_CUSTOM),
 			"Initialize the calling convention.\n"
 			"\n"
 			":param iterable arg_types: A list of :class:`DataType` values that define the argument types of a function.\n"
 			":param DataType return_type: The return type of a function.\n"
-			":param int alignment: The stack alignment."
+			":param int alignment: The stack alignment.\n"
+			":param Convention_t default_convention: The default convention for un override function."
 			)
 		)
 
@@ -844,14 +845,15 @@ void export_calling_convention(scope _memory)
 			&ICallingConventionWrapper::GetPopSize,
 			"Return the number of bytes that should be added to the stack to clean up."
 		)
-		
+
 		.def("get_argument_ptr",
-			&ICallingConventionWrapper::GetArgumentPtrWrapper,
+			&ICallingConventionWrapper::GetArgumentPtr,
 			(arg("index"), arg("registers")),
 			"Return a pointer to the argument at the given index.\n"
 			"\n"
 			":param int index: The index of the argument.\n"
-			":param Registers registers: A snapshot of all saved registers."
+			":param Registers registers: A snapshot of all saved registers.",
+			return_by_value_policy()
 		)
 
 		.def("argument_ptr_changed",
@@ -865,11 +867,12 @@ void export_calling_convention(scope _memory)
 		)
 
 		.def("get_return_ptr",
-			&ICallingConventionWrapper::GetReturnPtrWrapper,
+			&ICallingConventionWrapper::GetReturnPtr,
 			(arg("registers")),
 			"Return a pointer to the return value.\n"
 			"\n"
-			":param Registers registers: A snapshot of all saved registers."
+			":param Registers registers: A snapshot of all saved registers.",
+			return_by_value_policy()
 		)
 
 		.def("return_ptr_changed",

src/core/modules/memory/memory_wrap.h

@@ -32,6 +32,14 @@
 // ============================================================================
 // DynamicHooks
 #include "convention.h"
+#include "conventions/x86MsCdecl.h"
+#include "conventions/x86MsThiscall.h"
+#include "conventions/x86MsStdcall.h"
+#include "conventions/x86GccCdecl.h"
+#include "conventions/x86GccThiscall.h"
+
+// Memory
+#include "memory_function.h"
 
 // Utilities
 #include "memory_utilities.h"
@@ -48,14 +56,55 @@ using namespace boost::python;
 class ICallingConventionWrapper: public ICallingConvention, public wrapper<ICallingConvention>
 {
 public:
-	ICallingConventionWrapper(object oArgTypes, DataType_t returnType, int iAlignment=4)
+	ICallingConventionWrapper(object oArgTypes, DataType_t returnType, int iAlignment=4, Convention_t eDefaultConv=CONV_CUSTOM)
 		:ICallingConvention(ObjectToDataTypeVector(oArgTypes), returnType, iAlignment)
 	{
+#ifdef _WIN32
+		switch (eDefaultConv)
+		{
+		case CONV_CUSTOM:
+			break;
+		case CONV_CDECL:
+			m_pCallingConvention = new x86MsCdecl(m_vecArgTypes, m_returnType, m_iAlignment);
+			break;
+		case CONV_THISCALL:
+			m_pCallingConvention = new x86MsThiscall(m_vecArgTypes, m_returnType, m_iAlignment);
+			break;
+		case CONV_STDCALL:
+			m_pCallingConvention = new x86MsStdcall(m_vecArgTypes, m_returnType, m_iAlignment);
+			break;
+		default:
+			BOOST_RAISE_EXCEPTION(PyExc_ValueError, "Unsupported calling convention.")
+		}
+#else
+		switch (eDefaultConv)
+		{
+		case CONV_CUSTOM:
+			break;
+		case CONV_CDECL:
+			m_pCallingConvention = new x86GccCdecl(m_vecArgTypes, m_returnType, m_iAlignment);
+			break;
+		case CONV_THISCALL:
+			m_pCallingConvention = new x86GccThiscall(m_vecArgTypes, m_returnType, m_iAlignment);
+			break;
+		default:
+			BOOST_RAISE_EXCEPTION(PyExc_ValueError, "Unsupported calling convention.")
+		}
+#endif
+	}
+
+	~ICallingConventionWrapper()
+	{
+		delete m_pCallingConvention;
+		m_pCallingConvention = nullptr;
 	}
 
 	virtual std::list<Register_t> GetRegisters()
 	{
 		override get_registers = get_override("get_registers");
+		if (!get_registers && m_pCallingConvention) {
+			return m_pCallingConvention->GetRegisters();
+		}
 		CHECK_OVERRIDE(get_registers);
 
 		object registers = get_registers();
@@ -71,47 +120,60 @@ class ICallingConventionWrapper: public ICallingConvention, public wrapper<ICall
 	virtual int GetPopSize()
 	{
 		override get_pop_size = get_override("get_pop_size");
+		if (!get_pop_size && m_pCallingConvention) {
+			return m_pCallingConvention->GetPopSize();
+		}
 		CHECK_OVERRIDE(get_pop_size);
+
 		return get_pop_size();
 	}
-
-	virtual void* GetArgumentPtr(int iIndex, CRegisters* pRegisters)
-	{
-		CPointer* ptr = extract<CPointer*>(GetArgumentPtrWrapper(iIndex, pRegisters));
-		return (void *) ptr->m_ulAddr;
-	}
 
-	object GetArgumentPtrWrapper(int iIndex, CRegisters* pRegisters)
+	virtual void* GetArgumentPtr(int iIndex, CRegisters* pRegisters)
 	{
 		override get_argument_ptr = get_override("get_argument_ptr");
+		if (!get_argument_ptr && m_pCallingConvention) {
+			return m_pCallingConvention->GetArgumentPtr(iIndex, pRegisters);
+		}
 		CHECK_OVERRIDE(get_argument_ptr);
-		return get_argument_ptr(iIndex, ptr(pRegisters));
+
+		object argument_ptr = get_argument_ptr(iIndex, ptr(pRegisters));
+		CPointer* _ptr = extract<CPointer*>(argument_ptr);
+		return (void *) _ptr->m_ulAddr;
 	}
 
 	virtual void ArgumentPtrChanged(int iIndex, CRegisters* pRegisters, void* pArgumentPtr)
 	{
 		override argument_ptr_changed = get_override("argument_ptr_changed");
+		if (!argument_ptr_changed && m_pCallingConvention) {
+			m_pCallingConvention->ArgumentPtrChanged(iIndex, pRegisters, pArgumentPtr);
+			return;
+		}
 		CHECK_OVERRIDE(argument_ptr_changed);
 		argument_ptr_changed(iIndex, ptr(pRegisters), CPointer((unsigned long) pArgumentPtr));
 	}
 
 	virtual void* GetReturnPtr(CRegisters* pRegisters)
-	{
-		CPointer* ptr = extract<CPointer*>(GetReturnPtrWrapper(pRegisters));
-		return (void *) ptr->m_ulAddr;
-	}
-
-	object GetReturnPtrWrapper(CRegisters* pRegisters)
 	{
 		override get_return_ptr = get_override("get_return_ptr");
-		CHECK_OVERRIDE(get_return_ptr);
-		return get_return_ptr(ptr(pRegisters));
+		if (!get_return_ptr && m_pCallingConvention) {
+			return m_pCallingConvention->GetReturnPtr(pRegisters);
+		}
+		CHECK_OVERRIDE(get_return_ptr)
+
+		object return_ptr = get_return_ptr(ptr(pRegisters));
+		CPointer* _ptr = extract<CPointer*>(return_ptr);
+		return (void *) _ptr->m_ulAddr;
 	}
 
 	virtual void ReturnPtrChanged(CRegisters* pRegisters, void* pReturnPtr)
 	{
 		override return_ptr_changed = get_override("return_ptr_changed");
+		if (!return_ptr_changed && m_pCallingConvention) {
+			m_pCallingConvention->ReturnPtrChanged(pRegisters, pReturnPtr);
+			return;
+		}
 		CHECK_OVERRIDE(return_ptr_changed);
+
 		return_ptr_changed(ptr(pRegisters), CPointer((unsigned long) pReturnPtr));
 	}
 
@@ -125,6 +187,9 @@ class ICallingConventionWrapper: public ICallingConvention, public wrapper<ICall
 
 		return tuple(argumentTypes);
 	}
+
+public:
+	ICallingConvention* m_pCallingConvention = nullptr;
 };