Skip to content

Commit 72d8ff3

Browse files
committed
enhancement #9: gstreamer optimisations
1 parent 413e756 commit 72d8ff3

File tree

16 files changed

+113
-41
lines changed

16 files changed

+113
-41
lines changed

Plugin/android/Android.mk

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ GSTREAMER_PLUGINS := $(GSTREAMER_PLUGINS_CORE_CUSTOM) $(GSTREAMER_PLUGIN
4444
$(GSTREAMER_PLUGINS_ENCODING) \
4545
$(GSTREAMER_PLUGINS_SYS) \
4646
$(GSTREAMER_PLUGINS_PLAYBACK) \
47-
$(GSTREAMER_PLUGINS_EFFECTS_CUSTOM)
47+
$(GSTREAMER_PLUGINS_EFFECTS_CUSTOM) \
48+
rstracers
4849

4950
GSTREAMER_EXTRA_DEPS := gstreamer-video-1.0 glib-2.0 gstreamer-sdp-1.0 gstreamer-webrtc-nice-1.0 gstreamer-app-1.0 gstreamer-gl-1.0
5051

Plugin/android/Application.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ APP_STL := c++_shared
22
APP_PLATFORM := 31
33
APP_ABI := arm64-v8a
44
NDK_TARGET_LEVEL := android-31
5-
APP_OPTIM := release
5+
APP_OPTIM := debug
66
APP_BUILD_SCRIPT := Android.mk
77

88
#APP_PLATFORM = 15

Plugin/android/build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22

33
export GSTREAMER_ROOT_ANDROID=/mnt/data/fabien/android/gstreamer-1.0-android-universal-1.24.9/
4-
#export ANDROID_NDK_ROOT=/mnt/data/fabien/android/Sdk/ndk/25.2.9519653
4+
#export ANDROID_NDK_ROOT=/mnt/data/fabien/android/Sdk/ndk/29.0.13113456
55
export ANDROID_NDK_ROOT=/mnt/data/fabien/unity/2022.3.61f1/Editor/Data/PlaybackEngines/AndroidPlayer/NDK
66

77
$ANDROID_NDK_ROOT/ndk-build NDK_PROJECT_PATH=. NDK_APPLICATION_MK=Application.mk

Plugin/src/GstAVPipeline.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ GstElement* GstAVPipeline::add_webrtcsrc(GstElement* pipeline, const std::string
3939
void GstAVPipeline::webrtcbin_ready(GstElement* self, gchararray peer_id, GstElement* webrtcbin, gpointer udata)
4040
{
4141
Debug::Log("Configure webrtcbin", Level::Info);
42-
g_object_set(webrtcbin, "latency", 1, nullptr);
42+
g_object_set(webrtcbin, "latency", 10, nullptr);
4343
}
4444

4545
GstAVPipeline::GstAVPipeline(IUnityInterfaces* s_UnityInterfaces)

Plugin/src/GstAVPipelineOpenGLES.cpp

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ GstElement* GstAVPipelineOpenGLES::add_appsink(GstElement* pipeline)
184184
}
185185

186186
GstCaps* caps = gst_caps_from_string("video/x-raw(memory:GLMemory),format=RGBA,texture-target=2D");
187-
g_object_set(appsink, "caps", caps, "drop", true, "max-buffers", 1, "processing-deadline", (GstClockTime)1000000, nullptr);
187+
g_object_set(appsink, "caps", caps, "drop", true, "max-buffers", 1, "processing-deadline", (GstClockTime)10000000, nullptr);
188188
gst_caps_unref(caps);
189189

190190
gst_bin_add(GST_BIN(pipeline), appsink);
@@ -194,6 +194,12 @@ GstElement* GstAVPipelineOpenGLES::add_appsink(GstElement* pipeline)
194194
GstFlowReturn GstAVPipelineOpenGLES::on_new_sample(GstAppSink* appsink, gpointer user_data)
195195
{
196196
AppData* data = static_cast<AppData*>(user_data);
197+
if (data == nullptr)
198+
{
199+
Debug::Log("user data is null", Level::Error);
200+
return GST_FLOW_ERROR;
201+
}
202+
197203
GstSample* sample = gst_app_sink_pull_sample(appsink);
198204

199205
if (!sample)
@@ -227,9 +233,10 @@ void GstAVPipelineOpenGLES::on_pad_added(GstElement* src, GstPad* new_pad, gpoin
227233
if (g_str_has_prefix(pad_name, "video"))
228234
{
229235
Debug::Log("Adding video pad " + std::string(pad_name));
230-
// GstElement* queue = add_by_name(avpipeline->pipeline_, "queue");
231236
// decoder output texture-target=external-eos. converts to exture-target=2D
232237
GstElement* glcolorconvert = add_by_name(avpipeline->pipeline_, "glcolorconvert");
238+
GstElement* queue = add_by_name(avpipeline->pipeline_, "queue");
239+
g_object_set(queue, "max-size-buffers", 1, "max-size-bytes", 0, "max-size-time", (guint64)0, NULL);
233240
GstElement* appsink = add_appsink(avpipeline->pipeline_);
234241

235242
GstAppSinkCallbacks callbacks = {nullptr};
@@ -250,7 +257,7 @@ void GstAVPipelineOpenGLES::on_pad_added(GstElement* src, GstPad* new_pad, gpoin
250257
Debug::Log("Unknown video pad", Level::Warning);
251258
}
252259

253-
if (!gst_element_link_many(/*queue,*/ glcolorconvert, appsink, nullptr))
260+
if (!gst_element_link_many(glcolorconvert, queue, appsink, nullptr))
254261
{
255262
Debug::Log("Elements could not be linked.");
256263
}
@@ -262,8 +269,8 @@ void GstAVPipelineOpenGLES::on_pad_added(GstElement* src, GstPad* new_pad, gpoin
262269
}
263270
gst_object_unref(sinkpad);
264271

265-
// gst_element_sync_state_with_parent(queue);
266272
gst_element_sync_state_with_parent(glcolorconvert);
273+
gst_element_sync_state_with_parent(queue);
267274
gst_element_sync_state_with_parent(appsink);
268275
}
269276
else if (g_str_has_prefix(pad_name, "audio"))
@@ -274,10 +281,13 @@ void GstAVPipelineOpenGLES::on_pad_added(GstElement* src, GstPad* new_pad, gpoin
274281
GstElement* opusdec = add_by_name(avpipeline->pipeline_, "opusdec");
275282
GstElement* audioconvert = add_by_name(avpipeline->pipeline_, "audioconvert");
276283
GstElement* audioresample = add_by_name(avpipeline->pipeline_, "audioresample");
284+
GstElement* presink_queue = add_by_name(avpipeline->pipeline_, "queue");
285+
g_object_set(presink_queue, "max-size-buffers", 1, "max-size-bytes", 0, "max-size-time", (guint64)0, NULL);
277286
GstElement* autoaudiosink = add_by_name(avpipeline->pipeline_, "openslessink");
278-
g_object_set(autoaudiosink, "buffer-time", 10000, /*"processing-deadline", (GstClockTime)1000000,*/ nullptr);
287+
g_object_set(autoaudiosink, "buffer-time", 20000, "processing-deadline", (GstClockTime)1000000, nullptr);
279288

280-
if (!gst_element_link_many(queue, rtpopusdepay, opusdec, audioconvert, audioresample, autoaudiosink, nullptr))
289+
if (!gst_element_link_many(queue, rtpopusdepay, opusdec, audioconvert, audioresample, presink_queue, autoaudiosink,
290+
nullptr))
281291
{
282292
Debug::Log("Audio elements could not be linked.", Level::Error);
283293
}
@@ -289,11 +299,12 @@ void GstAVPipelineOpenGLES::on_pad_added(GstElement* src, GstPad* new_pad, gpoin
289299
}
290300
gst_object_unref(sinkpad);
291301

302+
gst_element_sync_state_with_parent(queue);
292303
gst_element_sync_state_with_parent(rtpopusdepay);
293304
gst_element_sync_state_with_parent(opusdec);
294-
gst_element_sync_state_with_parent(queue);
295305
gst_element_sync_state_with_parent(audioconvert);
296306
gst_element_sync_state_with_parent(audioresample);
307+
gst_element_sync_state_with_parent(presink_queue);
297308
gst_element_sync_state_with_parent(autoaudiosink);
298309
}
299310
g_free(pad_name);
@@ -306,6 +317,16 @@ GstAVPipelineOpenGLES::GstAVPipelineOpenGLES(IUnityInterfaces* s_UnityInterfaces
306317
{
307318
Debug::Log("Failed to load 'opengl' plugin", Level::Error);
308319
}
320+
preloaded_plugins.push_back(gst_plugin_load_by_name("opensles"));
321+
if (!preloaded_plugins.back())
322+
{
323+
Debug::Log("Failed to load 'opensles' plugin", Level::Error);
324+
}
325+
preloaded_plugins.push_back(gst_plugin_load_by_name("amcvideodec-omx"));
326+
if (!preloaded_plugins.back())
327+
{
328+
Debug::Log("Failed to load 'amcvideodec-omx' plugin", Level::Error);
329+
}
309330
}
310331

311332
GstBusSyncReply GstAVPipelineOpenGLES::busSyncHandler(GstBus* bus, GstMessage* msg, gpointer user_data)

Plugin/src/GstBasePipeline.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ gboolean GstBasePipeline::dumpLatencyCallback(GstBasePipeline* self)
140140
std::string msg = "Pipeline " + self->PIPENAME + " latency: live=" + std::to_string(live) +
141141
", min=" + std::to_string(min_latency) + ", max=" + std::to_string(max_latency);
142142
Debug::Log(msg);
143+
GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(self->pipeline_), GST_DEBUG_GRAPH_SHOW_ALL, self->PIPENAME.c_str());
143144
}
144145
gst_query_unref(query);
145146
return true;

Plugin/src/GstMicPipeline.cpp

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ void GstMicPipeline::CreatePipeline(const char* uri, const char* remote_peer_id)
2626
GstElement* queue = add_by_name(pipeline_, "queue");
2727
GstElement* opusenc = add_opusenc(pipeline_);
2828
GstElement* audio_caps_capsfilter = add_audio_caps_capsfilter(pipeline_);
29-
GstElement* webrtcsink = add_webrtcsink(pipeline_, uri);
29+
GstElement* webrtcsink = add_webrtcsink(pipeline_, uri, audiosrc);
3030

3131
if (!gst_element_link_many(audiosrc, queue, audioconvert, audioresample, webrtcdsp, opusenc, audio_caps_capsfilter,
3232
webrtcsink, nullptr))
@@ -100,7 +100,7 @@ GstElement* GstMicPipeline::add_audio_caps_capsfilter(GstElement* pipeline)
100100
return audio_caps_capsfilter;
101101
}
102102

103-
GstElement* GstMicPipeline::add_webrtcsink(GstElement* pipeline, const std::string& uri)
103+
GstElement* GstMicPipeline::add_webrtcsink(GstElement* pipeline, const std::string& uri, GstElement* audiosrc)
104104
{
105105
GstElement* webrtcsink = gst_element_factory_make("webrtcsink", nullptr);
106106
if (!webrtcsink)
@@ -132,7 +132,7 @@ GstElement* GstMicPipeline::add_webrtcsink(GstElement* pipeline, const std::stri
132132

133133
g_object_set(webrtcsink, "stun-server", nullptr, "do-retransmission", false, nullptr);
134134

135-
g_signal_connect(webrtcsink, "consumer-added", G_CALLBACK(consumer_added_callback), nullptr);
135+
g_signal_connect(webrtcsink, "consumer-added", G_CALLBACK(consumer_added_callback), audiosrc);
136136

137137
gst_bin_add(GST_BIN(pipeline), webrtcsink);
138138
return webrtcsink;
@@ -157,6 +157,7 @@ GstElement* GstMicPipeline::add_webrtcdsp(GstElement* pipeline)
157157
void GstMicPipeline::consumer_added_callback(GstElement* consumer_id, gchararray webrtcbin, GstElement* arg1, gpointer udata)
158158
{
159159
Debug::Log("Consumer added");
160+
GstElement* audiosrc = static_cast<GstElement*>(udata);
160161
GstIterator* sinks = gst_bin_iterate_sinks(GST_BIN(consumer_id));
161162
gboolean done = FALSE;
162163
while (!done)
@@ -181,6 +182,13 @@ void GstMicPipeline::consumer_added_callback(GstElement* consumer_id, gchararray
181182
// Free the item value
182183
g_value_unset(&item);
183184

185+
gint64 actual_buffer_time, actual_latency_time;
186+
187+
g_object_get(audiosrc, "actual-buffer-time", &actual_buffer_time, "actual-latency-time", &actual_latency_time,
188+
NULL);
189+
Debug::Log("Actual buffer time: " + std::to_string(actual_buffer_time) + " ns", Level::Info);
190+
Debug::Log("Actual latency time: " + std::to_string(actual_latency_time) + " ns", Level::Info);
191+
184192
break;
185193
}
186194
case GST_ITERATOR_RESYNC:

Plugin/src/GstMicPipeline.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,6 @@ class GstMicPipeline : public GstBasePipeline
2424

2525
static GstElement* add_opusenc(GstElement* pipeline);
2626
static GstElement* add_audio_caps_capsfilter(GstElement* pipeline);
27-
static GstElement* add_webrtcsink(GstElement* pipeline, const std::string& uri);
27+
static GstElement* add_webrtcsink(GstElement* pipeline, const std::string& uri, GstElement* audiosrc);
2828
static GstElement* add_webrtcdsp(GstElement* pipeline);
2929
};

Plugin/src/RenderingPlugin.cpp

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
This source code is licensed under the license found in the
33
LICENSE file in the root directory of this source tree. */
44

5+
#include "DebugLog.h"
56
#include "GstDataPipeline.h"
67
#include "GstMicPipeline.h"
78
#include "Unity/IUnityGraphics.h"
@@ -37,6 +38,7 @@ extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API CreateDevice()
3738

3839
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API CreatePipeline(const char* uri, const char* remote_peer_id)
3940
{
41+
Debug::Log("CreatePipeline", Level::Info);
4042
gstAVPipeline->CreatePipeline(uri, remote_peer_id);
4143
gstMicPipeline->CreatePipeline(uri, remote_peer_id);
4244
}
@@ -58,8 +60,10 @@ extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API ReleaseTexture(void*
5860

5961
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API DestroyPipeline()
6062
{
63+
Debug::Log("DestroyPipeline", Level::Info);
6164
gstAVPipeline->DestroyPipeline();
6265
gstMicPipeline->DestroyPipeline();
66+
gst_deinit();
6367
}
6468

6569
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API DestroyDataPipeline() { gstDataPipeline->DestroyPipeline(); }
@@ -81,13 +85,13 @@ extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API SendBytesChannelServi
8185
gstDataPipeline->send_byte_array_channel_service(data, size);
8286
}
8387

84-
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API SendBytesChannelReliableCommand(const unsigned char* data, size_t size)
88+
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API SendBytesChannelReliableCommand(const unsigned char* data,
89+
size_t size)
8590
{
8691
gstDataPipeline->send_byte_array_channel_command_reliable(data, size);
8792
}
8893

89-
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API SendBytesChannelLossyCommand(const unsigned char* data,
90-
size_t size)
94+
extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API SendBytesChannelLossyCommand(const unsigned char* data, size_t size)
9195
{
9296
gstDataPipeline->send_byte_array_channel_command_lossy(data, size);
9397
}
@@ -99,10 +103,9 @@ extern "C" void UNITY_INTERFACE_EXPORT UNITY_INTERFACE_API UnityPluginLoad(IUnit
99103
{
100104
// const char* internalStoragePath = getenv("EXTERNAL_STORAGE");
101105
// td::string logFilePath = std::string(internalStoragePath) + "/gstreamer.log";
102-
/*setenv("GST_DEBUG_FILE", "/storage/emulated/0/Android/data/com.DefaultCompany.UnityProject/files/gstreamer/gstreamer.log",
103-
1);*/
104-
setenv("GST_DEBUG_NO_COLOR", "1", 1);
105-
setenv("GST_DEBUG", "2", 1);
106+
// setenv("GST_DEBUG_FILE", "/storage/emulated/0/Android/data/com.DefaultCompany.UnityProject/files/gstreamer.log", 1);
107+
// setenv("GST_DEBUG_NO_COLOR", "1", 1);
108+
// setenv("GST_DEBUG", "4", 1);
106109
// gst_debug_set_threshold_for_name("basesink", GST_LEVEL_DEBUG);
107110

108111
#if UNITY_WIN

UnityProject/Assets/Scenes/gstreamer_scene.unity

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -566,7 +566,13 @@ MonoBehaviour:
566566
m_EditorClassIdentifier:
567567
leftRawImage: {fileID: 380550528}
568568
rightRawImage: {fileID: 31189211}
569-
ip_address: 10.0.1.36
569+
ip_address: 10.0.1.30
570+
event_OnPipelineRenderingRunning:
571+
m_PersistentCalls:
572+
m_Calls: []
573+
event_OnPipelineDataRunning:
574+
m_PersistentCalls:
575+
m_Calls: []
570576
--- !u!1 &1875013641
571577
GameObject:
572578
m_ObjectHideFlags: 0

UnityProject/Packages/com.pollenrobotics.gstreamerwebrtc/Runtime/Java/src/com/pollenrobotics/gstreamer/GstreamerActivity.java

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import android.widget.FrameLayout;
1212
import com.unity3d.player.UnityPlayerActivity;
1313
import org.freedesktop.gstreamer.GStreamer;
14+
import java.io.File;
1415

1516
public class GstreamerActivity extends UnityPlayerActivity {
1617

@@ -19,15 +20,43 @@ protected void onCreate(Bundle savedInstanceState) {
1920

2021
Log.d("GstreamerActivity", "onCreate called!");
2122

22-
try {
23+
try {
24+
//File externalFilesDir = getExternalFilesDir(null);
25+
String path = getExternalFilesDir(null).getAbsolutePath();
26+
Log.d("GstreamerActivity", "External files directory: " + path);
27+
/*if (externalFilesDir != null) {
28+
// Afficher le chemin du répertoire
29+
String directoryPath = externalFilesDir.getAbsolutePath();
30+
Log.d("DirectoryPath", "Chemin du répertoire: " + directoryPath);
31+
32+
// Afficher uniquement le nom du répertoire (dernière partie du chemin)
33+
String directoryName = externalFilesDir.getName();
34+
Log.d("DirectoryName", "Nom du répertoire: " + directoryName);
35+
} else {
36+
Log.e("DirectoryPath", "Impossible d'obtenir le répertoire de stockage externe.");
37+
}*/
2338
/*Os.setenv(
2439
"GST_DEBUG_FILE",
25-
"/storage/emulated/0/Android/data/com.DefaultCompany.UnityProject/files/gstreamer.log",
40+
path + "/gstreamer.log",
41+
//"/storage/emulated/0/Android/data/com.DefaultCompany.UnityProject/files/gstreamer.log",
2642
//"/sdcard/Android/data/com.DefaultCompany.UnityProject/files/gstreamer/gstreamer.log",
2743
true
2844
);*/
45+
Os.setenv(
46+
"GST_DEBUG_DUMP_DOT_DIR",
47+
path + "/log_dot",
48+
true
49+
);
50+
Os.setenv(
51+
"GST_TRACERS",
52+
//path + "/buffer_lateness.log",
53+
"buffer-lateness(file=\""+path+"/buffer_lateness.log\")",
54+
//"/storage/emulated/0/Android/data/com.DefaultCompany.UnityProject/files/buffer_lateness.log",
55+
//"/sdcard/Android/data/com.DefaultCompany.UnityProject/files/gstreamer/gstreamer.log",
56+
true
57+
);
2958
Os.setenv("GST_DEBUG_NO_COLOR", "1", true);
30-
Os.setenv("GST_DEBUG", "2", true);
59+
Os.setenv("GST_DEBUG", "3", true);
3160
} catch (ErrnoException ex) {
3261
Log.d(
3362
"OverrideActivity",
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:d31953b64cae0a7265f9a6528e39b95cc0717983ede7edb63256aed134d3c49b
3-
size 68496
2+
oid sha256:f9834da9d4873f26f9d1f56a2e2cb15f768be8df4a8b4eff4994ae324faed7e5
3+
size 113336
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
version https://git-lfs.github.com/spec/v1
2-
oid sha256:4b5560f258c67fc9eea5debf38bf867f49363a15cba9e0249fd70d9058a1ac54
3-
size 31407528
2+
oid sha256:ea00ec909e1a457a964489477f5a8cb3992c2d8dba051cc696ba8c78f7b3f859
3+
size 33273232

UnityProject/Packages/com.pollenrobotics.gstreamerwebrtc/Runtime/Scripts/GstreamerPlugin.cs

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,6 @@ void OnDestroy()
5151

5252
void Start()
5353
{
54-
55-
var path = Application.persistentDataPath + "/fake_log.log";
56-
Debug.Log("Log file path: " + path);
57-
File.WriteAllText(path, "test");
58-
5954
if (cleaning_thread != null)
6055
{
6156
cleaning_thread.Join();
@@ -70,8 +65,8 @@ void Start()
7065
//GStreamerRenderingPlugin has to run in main thread
7166
InitAV();
7267

73-
//init_thread = new Thread(InitData);
74-
//init_thread.Start();
68+
init_thread = new Thread(InitData);
69+
init_thread.Start();
7570
}
7671

7772
protected virtual void InitAV()

UnityProject/Packages/manifest.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"dependencies": {
33
"com.unity.collab-proxy": "2.6.0",
44
"com.unity.feature.development": "1.0.1",
5+
"com.unity.mobile.android-logcat": "1.4.5",
56
"com.unity.textmeshpro": "3.0.7",
67
"com.unity.timeline": "1.7.6",
78
"com.unity.ugui": "1.0.0",

0 commit comments

Comments
 (0)