Added OpenCV and WebRTC integration sample app

Author: Ihor Lutsyk

src/webrtc/samples/CMakeLists.txt

@@ -4,4 +4,5 @@ if(HAVE_WEBRTC)
     add_subdirectory(webrtcrecorder)
   endif()
   add_subdirectory(webrtccapturer)
+  add_subdirectory(opencvanalyzer)
 endif()

src/webrtc/samples/opencvanalyzer/CMakeLists.txt

@@ -0,0 +1,8 @@
+include_dependency(WebRTC REQUIRED)
+include_dependency(OpenCV REQUIRED)
+
+define_sourcey_module_sample(opencvanalyzer base crypto av net http util json socketio symple webrtc)
+
+configure_file(network/mobilenetssd.caffemodel network/mobilenetssd.caffemodel COPYONLY)
+
+configure_file(network/mobilenetssd.prototxt.txt network/mobilenetssd.prototxt.txt COPYONLY)

src/webrtc/samples/opencvanalyzer/README.md

@@ -0,0 +1,30 @@
+# WebRTC OpencCV Analyzer
+
+This sample app showcases how to pass live WebRTC video stream to OpenCV to perform object detection.
+Demo has next actors:
+1. Client - web browser, which is WebRTC source and video player as well (local loopback)
+1. Libsourcey server app that connects as WebRTC peer to client's stream.
+
+Libsourcey server app acts as WebRTC listener, OpenCV processor and WebSocket server that sends objects detection data back to a client.
+The client app draws received object detections (through separate WebSocket channel) using js to render detections over video loopback HTML element.
+This is also interesting demo how to avoid delays in video stream and deliver analytics by separate channel, without extra encoding on server side.
+
+## Installation
+
+1. Compile LibSourcey with WebRTC and OpenCV support (`WITH_WEBRTC=ON WITH_OPENCV=ON`) and samples (`BUILD_SAMPLES_webrtc=ON`) enabled. See the main README for help: https://github.com/sourcey/libsourcey#installing-on-linux
+1. Run the `opencvanalyzer` sample application from the `build` directory.
+You can rewrite default network settings by arguments, run `opencvanalyzer -h` for help dialog.
+1. Install and run the Node.js client application:
+
+~~~ bash
+cd <libsourcey>/src/webrtc/samples/opencvanalyzer/client
+npm install
+node app
+~~~
+
+4. Finally, point your browser to: `http://localhost:4499`
+
+IMPORTANT NOTICE!
+Chrome: enable mixed content: `chrome --allow-running-insecure-content`
+Firefox: this fix should be applied, until it is not merged to upstream project:
+https://github.com/sourcey/libsourcey/pull/205

src/webrtc/samples/opencvanalyzer/client/app.js

@@ -0,0 +1,39 @@
+//
+/// Setup the Symple server
+var fs = require('fs');
+var Symple = require('symple');
+var sy = new Symple();
+sy.loadConfig(__dirname + '/symple.json'); // see symple.json for options
+sy.init();
+console.log('Symple server listening on port ' + sy.config.port);
+
+
+//
+/// Setup the demo client web server
+
+var express = require('express'),
+  path = require('path'),
+  app = express(),
+  serverPort = parseInt(sy.config.port)
+clientPort = serverPort - 1;
+
+app.set('port', clientPort);
+app.set('view engine', 'ejs');
+app.set('views', __dirname + '/');
+app.use(express.static(__dirname + '/assets'));
+
+
+app.get('/', function (req, res) {
+  res.render('index', {
+    port: serverPort,
+    peer: {
+      user: 'demo',
+      name: 'Demo User',
+      group: 'public'
+    }
+  });
+});
+
+app.listen(app.get('port'), function () {
+  console.log('Web server listening on port ' + app.get('port'));
+});

src/webrtc/samples/opencvanalyzer/client/assets/detection.js

@@ -0,0 +1,71 @@
+var canvas, ctx;
+// hardcoded value for video of 640x480, TODO: make it generic
+var detectionSize = 480;
+var webSocketHost = 'ws://0.0.0.0:1234/';
+var clearingTimeout;
+function init() {
+  // init canvas
+  canvas = document.getElementById("detectionCanvas");
+  ctx = canvas.getContext("2d");
+  ctx.lineWidth="3";
+  ctx.strokeStyle="red";
+  ctx.font="18px Georgia";
+  ctx.fillStyle = "red";
+
+  // init websocket
+  initWebSocketConnection();
+}
+
+function initWebSocketConnection() {
+  websocket = new WebSocket(webSocketHost);
+  websocket.onopen = function() { console.log('WebSocket Data Stream Connected'); };
+  websocket.onmessage = function(evt) {
+    console.log('Detection Data Received:', evt.data);
+    drawDetections(JSON.parse(evt.data));
+  };
+  websocket.onerror = function(evt) {
+    console.log('WebSocket ERROR: ' + evt.data);
+
+    // close websocket if opened
+    if(websocket.readyState < 2){
+      websocket.close();
+    }
+
+    // try to reinitialize websocket connection in one second
+    setTimeout(initWebSocketConnection, 1000);
+  };
+}
+
+function drawDetections(data) {
+  // remote handler for clearing data if there are no detections from server
+  if (clearingTimeout){
+    clearTimeout(clearingTimeout);
+  }
+
+  // clear previous drawings
+  clearDetections();
+
+  //draw detections one by one
+  ctx.beginPath();
+  for(var i=0; i<data.detections.length; i++) {
+    var detection = data.detections[i];
+
+    ctx.rect(detection.box.leftBottomX + 10,
+      detection.box.leftBottomY - 10,
+      detection.box.rightTopX - detection.box.leftBottomX - 30,
+      detection.box.rightTopY - detection.box.leftBottomY -  10);
+    ctx.fillText(detection.class + ' ' + detection.confidence,
+      detection.box.leftBottomX + 15, detection.box.leftBottomY + 15);
+    ctx.stroke();
+  }
+  ctx.closePath();
+
+  // set timer for clearing previous detection drawings
+  // if there are no new detections from server
+  clearingTimeout = setTimeout(clearDetections, 1000);
+}
+
+function clearDetections() {
+  ctx.clearRect(0, 0, canvas.width, canvas.height);
+}
+window.addEventListener("load", init, false);

src/webrtc/samples/opencvanalyzer/client/assets/main.css

@@ -0,0 +1,18 @@
+body > .container {
+    padding-top: 30px;
+}
+
+#webrtc-video .video-player {
+    margin-bottom: 1rem;
+}
+
+#webrtc-video .symple-player {
+    min-height: 480px;
+}
+
+/*This is hardcoded value for video of 640x480, TODO: make it generic*/
+#detectionCanvas {
+    position:absolute;
+    z-index: 1;
+    margin-left:-240px;
+}

src/webrtc/samples/opencvanalyzer/client/assets/main.js

@@ -0,0 +1,95 @@
+$(document).ready(function() {
+  var client, player, remotePeer, initialized = false;
+
+  //
+  // Initialize the Symple WebRTC player
+
+  player = new Symple.Player({
+    element: '#webrtc-video .video-player',
+    engine: 'WebRTC',
+    initiator: true,
+    rtcConfig: WEBRTC_CONFIG,
+    iceMediaConstraints: {
+      'mandatory': {
+        'OfferToReceiveAudio': false,
+        'OfferToReceiveVideo': false
+      }
+    },
+    onStateChange: function(player, state) {
+      player.displayStatus(state);
+    }
+  });
+
+  function startPlaybackAndRecording() {
+    // player.setup();
+    player.play(); //{ localMedia: true, disableAudio: false, disableVideo: false }
+    player.engine.sendLocalSDP = function(desc) {
+      console.log('Send offer:', JSON.stringify(desc));
+      client.send({
+        to: remotePeer,
+        type: 'message',
+        offer: desc
+      });
+    };
+
+    player.engine.sendLocalCandidate = function(cand) {
+      client.send({
+        to: remotePeer,
+        type: 'message',
+        candidate: cand
+      });
+    }
+  }
+
+  //
+  // Initialize the Symple client
+
+  client = new Symple.Client(CLIENT_OPTIONS);
+
+  client.on('announce', function(peer) {
+    console.log('Authentication success:', peer);
+  });
+
+  client.on('addPeer', function(peer) {
+    console.log('Adding peer:', peer);
+
+    if (peer.user == 'videorecorder' &&
+      !initialized) {
+      initialized = true;
+      remotePeer = peer; //m.from;
+      startPlaybackAndRecording();
+    }
+  });
+
+  client.on('removePeer', function(peer) {
+    console.log('Removing peer:', peer);
+  });
+
+  client.on('message', function(m) {
+    // console.log('Recv message:', m)
+    if (remotePeer && remotePeer.id != m.from.id) {
+      console.log('Dropping message from unknown peer', m);
+      return;
+    }
+    if (m.offer) {
+      alert('Unexpected offer for one-way streaming');
+    }
+    else if (m.answer) {
+      console.log('Reieve answer:', JSON.stringify(m.answer));
+      player.engine.recvRemoteSDP(m.answer);
+    }
+    else if (m.candidate) {
+      player.engine.recvRemoteCandidate(m.candidate);
+    }
+  });
+
+  client.on('disconnect', function() {
+    console.log('Disconnected from server')
+  });
+
+  client.on('error', function(error, message) {
+    console.log('Peer error:', error, message)
+  });
+
+  client.connect();
+});