微软Rocket视频分析平台学习（2）

代码地址https://github.com/microsoft/Microsoft-Rocket-Video-Analytics-Platform

rocket视频分析平台说明ppt  Rocket-features-and-pipelines.pdf.

1 拉取镜像

docker pull ycshu086/rocket-sample-edgeonly:0.1

2 以拉取下来的镜像创建容器，测试rocket分析平台

docker run --runtime=nvidia -v <local directory>:/app/output ycshu086/rocket-sample-edgeonly:0.1 sample.mp4 sample.txt 1 1 car

运行结果

<local directory>:/app/output  输出结果

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.

namespace Utils.Config
{
    public static class OutputFolder
    {
        public static string OutputFolderAll { get; set; } = "../../output_all/";

        public static string OutputFolderBGSLine { get; set; } = "../../output_bgsline/";

        public static string OutputFolderLtDNN { get; set; } = "../../output_ltdnn/";

        public static string OutputFolderCcDNN { get; set; } = "../../output_ccdnn/";

        public static string OutputFolderAML { get; set; } = "../../output_aml/";

        public static string OutputFolderFrameDNNDarknet { get; set; } = "../../output_framednndarknet/";

        public static string OutputFolderFrameDNNTF { get; set; } = "../../output_framednntf/";
    }
}

运行分析

首先

在VideoPipelineCore-App.config中设置管道配置PplConfig。我们在代码中预编译了六个配置。管道描述也包含在Rocket-features-and-pipelines.pdf中。

0：  基于线触发

1： 每帧上的Darknet Yolo v3（幻灯片7）

2： 每帧上的TensorFlow FastRCNN（幻灯片#8）

3： 基于背景减法（BGS）的预过滤->Darknet  Tiny Yolo->Darknet Yolo v3（幻灯片#9）

4： BGS预过滤->Darknet Tiny Yolo->数据库（Azure上的ArangoDB和blob存储）（幻灯片10）

5： BGS预过滤->TensorFlow Fast R-CNN->Azure机器学习（云）（幻灯片11）

<?xml version="1.0" encoding="utf-8" ?>
<configuration>
  <appSettings>
    <!--Pipeline Configuration-->
    <!--
    0: Line-based alerting
    1: Darknet Yolo v3 on every frame
    2: TensorFlow FastRCNN on every frame
    3: BGS early filtering -> Darknet Tiny Yolo -> Darknet Yolo v3
    4: BGS early filtering -> Darknet Tiny Yolo -> Database (ArangoDB and blob storage on Azure)
    5: BGS early filtering -> TensorFlow Fast R-CNN -> Azure Machine Learning (cloud)
    -->
    <add key="PplConfig" value="1" />
    
    <!--Azure Storage Connection-->
    <add key="StorageConnectionString" value="" />
    
    <!--Azure DB Connection-->
    <add key="DBSERVER" value="" />
    <add key="DBCRED" value="" />
    <add key="DBNAME" value="" />

    <!--Azure Machine Learning Service Connection-->
    <add key="AMLHost" value="" />
    <add key="AMLSSL" value="" />
    <add key="AMLAuthKey" value="" />
    <add key="AMLServiceID" value="" />
  </appSettings>
</configuration>

（可选）如果PplConfig设置为4或5，请设置您自己的数据库和Azure机器学习服务。

通过创建虚拟机在Azure上部署SQL数据库（如MySQL）或NoSQL数据库（如ArangoDB）。在App.Config中为Rocket提供数据库设置（如服务器名、用户名、凭据等）。您还可以设置云存储（例如，Azure Blob存储）来存储图像/视频。在管道4中，Rocket将检测图像发送到Azure存储帐户，并将元数据发送到Azure数据库。

深度学习模型部署到Azure（例如，使用Azure Kubernetes服务或AKS）以使用GPU或FPGA进行推理。将模型部署为web服务后，请在App.Config中向VAP提供主机URL、密钥和服务ID。Rocker将处理本地模块和云服务之间的通信。

运行代码分析

基于线触发的目标检测（另一种是基于物体边缘的目标检测）

<video_file/camera_url> <line_detection_config_file> <sampling_factor> <resolution_factor> <object_category>.

测试代码sample.mp4 sample.txt 1 1 car

视频流地址或者本地视频 cofig文件 采样率 分辨率 目标类别

<line_name> <line_id> <x_1> <y_1> <x_2> <y_2> <overlap_threshold>

sample.txt  Left-entrance    1    209    101    266    101    0.3

pipeline 主代码   Program.cs

            //parse arguments
            if (args.Length < 5)
            {
                Console.WriteLine(args.Length);
                Console.WriteLine("Usage: <exe> <video url> <cfg file> <samplingFactor> <resolutionFactor> <category1> <category2> ...");
                return;
            }

对应<video_file/camera_url> <line_detection_config_file> <sampling_factor> <resolution_factor> <object_category>.

  string videoUrl = args[0];
            bool isVideoStream;
            if (videoUrl.Substring(0, 4) == "rtmp" || videoUrl.Substring(0, 4) == "http" || videoUrl.Substring(0, 3) == "mms" || videoUrl.Substring(0, 4) == "rtsp")
            {
                isVideoStream = true;
            }
            else
            {
                isVideoStream = false;
                videoUrl = @"./media/" + args[0];
            }

可以看出可以用网络视频流 "rtmp" "http" "mms"  "rtsp"或者本地视频./media/作为输入进行视频分析

 string lineFile = @"./cfg/" + args[1];
            int SAMPLING_FACTOR = int.Parse(args[2]);
            double RESOLUTION_FACTOR = double.Parse(args[3]);

            Dictionary<string, int> category = new Dictionary<string, int>();
            for (int i = 4; i < args.Length; i++)
            {
                category.Add(args[i], 0);
            }

输入参数 <line_detection_config_file> <sampling_factor> <resolution_factor> <object_category>.

//initialize pipeline settings
            int pplConfig = Convert.ToInt16(ConfigurationManager.AppSettings["PplConfig"]);
            bool loop = false;
            bool displayRawVideo = false;
            bool displayBGSVideo = false;
            Utils.Utils.cleanFolder(@OutputFolder.OutputFolderAll);

初始化pipeline设置

//create pipeline components (initialization based on pplConfig)

            //-----Decoder-----
            Decoder.Decoder decoder = new Decoder.Decoder(videoUrl, loop);

            //-----Background Subtraction-based Detector-----
            BGSObjectDetector.BGSObjectDetector bgs = new BGSObjectDetector.BGSObjectDetector();

            //-----Line Detector-----
            Detector lineDetector = new Detector(SAMPLING_FACTOR, RESOLUTION_FACTOR, lineFile, displayBGSVideo);
            Dictionary<string, bool> occupancy = null;
            List<Tuple<string, int[]>> lines = lineDetector.multiLaneDetector.getAllLines();

            //-----LineTriggeredDNN (TensorFlow)-----
            LineTriggeredDNNTF ltDNNTF = null;
            List<Item> ltDNNItemListTF = null;
            if (new int[] { 5 }.Contains(pplConfig))
            {
                ltDNNTF = new LineTriggeredDNNTF(lines);
                ltDNNItemListTF = new List<Item>();
            }

            //-----DNN on every frame (TensorFlow)-----
            FrameDNNTF frameDNNTF = null;
            List<Item> frameDNNTFItemList = null;
            if (new int[] { 2 }.Contains(pplConfig))
            {
                frameDNNTF = new FrameDNNTF(lines);
                frameDNNTFItemList = new List<Item>();
                Utils.Utils.cleanFolder(@OutputFolder.OutputFolderFrameDNNTF);
            }

            //-----Call ML models deployed on Azure Machine Learning Workspace-----
            AMLCaller amlCaller = null;
            List<bool> amlConfirmed;
            if (new int[] { 5 }.Contains(pplConfig))
            {
                amlCaller = new AMLCaller(ConfigurationManager.AppSettings["AMLHost"],
                Convert.ToBoolean(ConfigurationManager.AppSettings["AMLSSL"]),
                ConfigurationManager.AppSettings["AMLAuthKey"],
                ConfigurationManager.AppSettings["AMLServiceID"]);
            }

            //-----Write to DB-----
            List<Item> ItemList = null;

            int frameIndex = 0;
            int videoTotalFrame = 0;
            if (!isVideoStream)
                videoTotalFrame = decoder.getTotalFrameNum() - 1; //skip the last frame which could be wrongly encoded from vlc capture

创建pipeline组件，包括解码（Decoder）、基于背景差分检测（Background Subtraction-based Detector）、直线检测（Line Detector）、线触发dnn（LineTriggeredDNN (TensorFlow)）、每帧图像的dnn（DNN on every frame (TensorFlow)）、调用部署在Azure机器学习工作区上的ML模型（Call ML models deployed on Azure Machine Learning Workspace）、写入db数据库（Write to DB）

//RUN PIPELINE 
            DateTime startTime = DateTime.Now;
            DateTime prevTime = DateTime.Now;
            while (true)
            {
                if (!loop)
                {
                    if (!isVideoStream && frameIndex >= videoTotalFrame)
                    {
                        break;
                    }
                }

                //decoder
                Mat frame = decoder.getNextFrame();

                
                //frame pre-processor
                frame = FramePreProcessor.PreProcessor.returnFrame(frame, frameIndex, SAMPLING_FACTOR, RESOLUTION_FACTOR, displayRawVideo);
                frameIndex++;
                if (frame == null) continue;
                //Console.WriteLine("Frame ID: " + frameIndex);


                //background subtractor
                Mat fgmask = null;
                List<Box> foregroundBoxes = bgs.DetectObjects(DateTime.Now, frame, frameIndex, out fgmask);
                

                //line detector
                if (new int[] { 0, 4, 5 }.Contains(pplConfig))
                {
                    occupancy = lineDetector.updateLineOccupancy(frame, frameIndex, fgmask, foregroundBoxes);
                }


                //cheap DNN
                if (new int[] { 4, 5 }.Contains(pplConfig))
                {
                    ltDNNItemListTF = ltDNNTF.Run(frame, frameIndex, occupancy, lines, category);
                    ItemList = ltDNNItemListTF;
                }


                //frame DNN TF
                if (new int[] { 2 }.Contains(pplConfig))
                {
                    frameDNNTFItemList = frameDNNTF.Run(frame, frameIndex, category, System.Drawing.Brushes.Pink, 0.2);
                    ItemList = frameDNNTFItemList;
                }


                //Azure Machine Learning
                if (new int[] { 5 }.Contains(pplConfig))
                {
                    amlConfirmed = AMLCaller.Run(frameIndex, ItemList, category).Result;
                }


                //DB Write
                if (new int[] { 4 }.Contains(pplConfig))
                {
                    Position[] dir = { Position.Unknown, Position.Unknown }; // direction detection is not included
                    DataPersistence.PersistResult("test", videoUrl, 0, frameIndex, ItemList, dir, "Cheap", "Heavy", // ArangoDB database
                                                            "test"); // Azure blob
                }

运行pipeline