enhancement #9: microphone support

Author: FabienDanieau

Plugin/android/Android.mk

@@ -6,7 +6,7 @@ include $(CLEAR_VARS)
 #SYSROOT  :=  $(abspath $(LOCAL_PATH))
 
 LOCAL_MODULE    := UnityGStreamerPlugin
-LOCAL_SRC_FILES := ../src/DebugLog.cpp ../src/RenderingPlugin.cpp ../src/GstDataPipeline.cpp ../src/GstBasePipeline.cpp ../src/GstAVPipeline.cpp ../src/GstAVPipelineOpenGLES.cpp
+LOCAL_SRC_FILES := ../src/DebugLog.cpp ../src/RenderingPlugin.cpp ../src/GstDataPipeline.cpp ../src/GstBasePipeline.cpp ../src/GstAVPipeline.cpp ../src/GstAVPipelineOpenGLES.cpp ../src/GstMicPipeline.cpp
 
 LOCAL_SHARED_LIBRARIES := gstreamer_android
 LOCAL_LDLIBS := -llog -landroid -lEGL -lGLESv2

Plugin/src/GstMicPipeline.cpp

@@ -2,28 +2,35 @@
  This source code is licensed under the license found in the
  LICENSE file in the root directory of this source tree. */
 
-#include "DebugLog.h"
 #include "GstMicPipeline.h"
+#include "DebugLog.h"
+#include "Unity/PlatformBase.h"
 
 GstMicPipeline::GstMicPipeline() : GstBasePipeline("MicPipeline") {}
 
-void GstMicPipeline::CreatePipeline(const char* uri, const char* remote_peer_id) 
+void GstMicPipeline::CreatePipeline(const char* uri, const char* remote_peer_id)
 {
     Debug::Log("GstMicPipeline create pipeline", Level::Info);
     Debug::Log(uri, Level::Info);
     Debug::Log(remote_peer_id, Level::Info);
 
     GstBasePipeline::CreatePipeline();
 
-    GstElement* wasapi2src = add_wasapi2src(pipeline_);
+#if UNITY_WIN
+    GstElement* audiosrc = add_wasapi2src(pipeline_);
+#elif UNITY_ANDROID
+    GstElement* audiosrc = add_by_name(pipeline_, "openslessrc");
+#endif
     GstElement* webrtcdsp = add_webrtcdsp(pipeline_);
-    GstElement* audioconvert = add_audioconvert(pipeline_);
-    GstElement* queue = add_queue(pipeline_);
+    GstElement* audioconvert = add_by_name(pipeline_, "audioconvert");
+    GstElement* audioresample = add_by_name(pipeline_, "audioresample");
+    GstElement* queue = add_by_name(pipeline_, "queue");
     GstElement* opusenc = add_opusenc(pipeline_);
     GstElement* audio_caps_capsfilter = add_audio_caps_capsfilter(pipeline_);
     GstElement* webrtcsink = add_webrtcsink(pipeline_, uri);
 
-    if (!gst_element_link_many(wasapi2src, queue, audioconvert, webrtcdsp, opusenc, audio_caps_capsfilter, webrtcsink, nullptr))
+    if (!gst_element_link_many(audiosrc, queue, audioconvert, audioresample, webrtcdsp, opusenc, audio_caps_capsfilter,
+                               webrtcsink, nullptr))
     {
         Debug::Log("Audio sending elements could not be linked.", Level::Error);
     }
@@ -45,32 +52,6 @@ GstElement* GstMicPipeline::add_wasapi2src(GstElement* pipeline)
     return wasapi2src;
 }
 
-GstElement* GstMicPipeline::add_queue(GstElement* pipeline)
-{
-    GstElement* queue = gst_element_factory_make("queue", nullptr);
-    if (!queue)
-    {
-        Debug::Log("Failed to create queue", Level::Error);
-        return nullptr;
-    }
-
-    gst_bin_add(GST_BIN(pipeline), queue);
-    return queue;
-}
-
-GstElement* GstMicPipeline::add_audioconvert(GstElement* pipeline)
-{
-    GstElement* audioconvert = gst_element_factory_make("audioconvert", nullptr);
-    if (!audioconvert)
-    {
-        Debug::Log("Failed to create audioconvert", Level::Error);
-        return nullptr;
-    }
-
-    gst_bin_add(GST_BIN(pipeline), audioconvert);
-    return audioconvert;
-}
-
 GstElement* GstMicPipeline::add_opusenc(GstElement* pipeline)
 {
     GstElement* opusenc = gst_element_factory_make("opusenc", nullptr);
@@ -80,7 +61,7 @@ GstElement* GstMicPipeline::add_opusenc(GstElement* pipeline)
         return nullptr;
     }
 
-    g_object_set(opusenc, "audio-type", "restricted-lowdelay", "frame-size", 10, nullptr);
+    g_object_set(opusenc, "audio-type", /*"restricted-lowdelay"*/ 2051, "frame-size", 10, nullptr);
 
     gst_bin_add(GST_BIN(pipeline), opusenc);
     return opusenc;
@@ -134,7 +115,7 @@ GstElement* GstMicPipeline::add_webrtcsink(GstElement* pipeline, const std::stri
     gst_structure_free(meta_structure);
     g_value_unset(&meta_value);
 
-    g_object_set(webrtcsink, "stun-server", nullptr, "do-restransmission", false, nullptr);
+    g_object_set(webrtcsink, "stun-server", nullptr, "do-retransmission", false, nullptr);
 
     g_signal_connect(webrtcsink, "consumer-added", G_CALLBACK(consumer_added_callback), nullptr);
 

Plugin/src/GstMicPipeline.h

@@ -13,10 +13,8 @@ class GstMicPipeline : public GstBasePipeline
 
     void CreatePipeline(const char* uri, const char* remote_peer_id);
 
-private:  
+private:
     static void consumer_added_callback(GstElement* consumer_id, gchararray webrtcbin, GstElement* arg1, gpointer udata);
-    static GstElement* add_queue(GstElement* pipeline);
-    static GstElement* add_audioconvert(GstElement* pipeline);
     static GstElement* add_wasapi2src(GstElement* pipeline);
     static GstElement* add_opusenc(GstElement* pipeline);
     static GstElement* add_audio_caps_capsfilter(GstElement* pipeline);

Plugin/src/RenderingPlugin.cpp

@@ -3,15 +3,14 @@
  LICENSE file in the root directory of this source tree. */
 
 #include "GstDataPipeline.h"
+#include "GstMicPipeline.h"
 #include "Unity/IUnityGraphics.h"
 #include "Unity/PlatformBase.h"
-//#include <assert.h>
+// #include <assert.h>
 
 #if UNITY_WIN
 #include "GstAVPipelineD3D11.h"
-#include "GstMicPipeline.h"
 
-static std::unique_ptr<GstMicPipeline> gstMicPipeline = nullptr;
 static std::unique_ptr<GstAVPipelineD3D11> gstAVPipeline = nullptr;
 
 #elif UNITY_ANDROID
@@ -33,6 +32,7 @@ static jobject surface_plugin = nullptr;
 
 #endif
 
+static std::unique_ptr<GstMicPipeline> gstMicPipeline = nullptr;
 static std::unique_ptr<GstDataPipeline> gstDataPipeline = nullptr;
 
 extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API CreateDevice()
@@ -55,9 +55,7 @@ extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API SetSurface(jobject su
 extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API CreatePipeline(const char* uri, const char* remote_peer_id)
 {
     gstAVPipeline->CreatePipeline(uri, remote_peer_id);
-#if UNITY_WIN
     gstMicPipeline->CreatePipeline(uri, remote_peer_id);
-#endif
 }
 
 extern "C" UNITY_INTERFACE_EXPORT void* UNITY_INTERFACE_API CreateTexture(unsigned int width, unsigned int height, bool left)
@@ -76,9 +74,7 @@ extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API ReleaseTexture(void*
 extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API DestroyPipeline()
 {
     gstAVPipeline->DestroyPipeline();
-#if UNITY_WIN
     gstMicPipeline->DestroyPipeline();
-#endif
 }
 
 extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API DestroyDataPipeline() { gstDataPipeline->DestroyPipeline(); }
@@ -127,12 +123,11 @@ extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API UnityPluginLoad(IUnit
 #if UNITY_WIN
     gst_init(nullptr, nullptr);
     gstAVPipeline = std::make_unique<GstAVPipelineD3D11>(unityInterfaces);
-    gstMicPipeline = std::make_unique<GstMicPipeline>();
 #elif UNITY_ANDROID
     // gst_init done in the java side
     gstAVPipeline = std::make_unique<GstAVPipelineOpenGLES>(unityInterfaces);
 #endif
-
+    gstMicPipeline = std::make_unique<GstMicPipeline>();
     gstDataPipeline = std::make_unique<GstDataPipeline>();
 }
 

UnityProject/Assets/Plugins/Android/AndroidManifest.xml

@@ -12,4 +12,5 @@
         </intent-filter>
     </activity>
   </application>
+  <uses-permission android:name="android.permission.RECORD_AUDIO"/>
 </manifest>

UnityProject/Packages/com.pollenrobotics.gstreamerwebrtc/Runtime/Java/src/com/pollenrobotics/gstreamer/RenderingCallbackManager.java

@@ -1,6 +1,7 @@
 package com.pollenrobotics.gstreamer;
 
 import java.util.ArrayList;
+import android.util.Log;
 
 public class RenderingCallbackManager {
     @SuppressWarnings("JniMissingFunction")

UnityProject/Packages/com.pollenrobotics.gstreamerwebrtc/Runtime/Java/src/com/pollenrobotics/gstreamer/UnityRenderTexture.java

@@ -81,7 +81,7 @@ public Surface GetSurface(){
     @Override
     public void onFrameAvailable(SurfaceTexture surfaceTexture) {
         RenderingCallbackManager.Instance().SubscribeOneShot(eventCode->{
-            surfaceTexture.updateTexImage();
+                surfaceTexture.updateTexImage();            
         });
     }
 }

UnityProject/Packages/com.pollenrobotics.gstreamerwebrtc/Runtime/Plugins/Android/AndroidManifest.xml

@@ -12,4 +12,5 @@
         </intent-filter>
     </activity>
   </application>
+  <uses-permission android:name="android.permission.RECORD_AUDIO"/>
 </manifest>

UnityProject/Packages/com.pollenrobotics.gstreamerwebrtc/Runtime/Plugins/arm64-v8a/libUnityGStreamerPlugin.so

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f49406b7de9ab8a4283dd32b397d7a6aaac6fa296fcfff7bb40aed887fc926f0
-size 95624
+oid sha256:478baa7e5a9c46f36421df63c64c809406bbe7baeec0487568625df2fe7c0ff0
+size 105560

UnityProject/Packages/com.pollenrobotics.gstreamerwebrtc/Runtime/Scripts/GstreamerRenderingPlugin.cs

@@ -8,7 +8,7 @@ LICENSE file in the root directory of this source tree. */
 using UnityEngine.Rendering;
 using UnityEngine.Events;
 using System.Collections;
-
+using UnityEngine.Android;
 
 namespace GstreamerWebRTC
 {
@@ -96,6 +96,8 @@ public GStreamerRenderingPlugin(string ip_address, ref Texture leftTexture, ref
             _command = new CommandBuffer();
 
 #if UNITY_ANDROID
+            CheckMicPermission();
+            
             _command.IssuePluginEvent(GetRenderEventFunc(), 1);
             Camera.main.AddCommandBuffer(CameraEvent.BeforeSkybox, _command);
 
@@ -122,6 +124,19 @@ public GStreamerRenderingPlugin(string ip_address, ref Texture leftTexture, ref
         }
 
 #if UNITY_ANDROID
+        private void CheckMicPermission()
+        {
+            if (Permission.HasUserAuthorizedPermission(Permission.Microphone))
+            {
+                Debug.Log("Microphone permission already granted");
+            }
+            else
+            {
+                Permission.RequestUserPermission(Permission.Microphone);
+            }
+
+        }
+
         public bool IsNativePtrSet()
         {
             return nativeTexPtrSet;