C# Onnx 使用onnxruntime部署实时视频帧插值

目录

介绍

效果

模型信息

项目

代码

下载

C# Onnx 使用onnxruntime部署实时视频帧插值

介绍

github地址：https://github.com/google-research/frame-interpolation

FILM: Frame Interpolation for Large Motion, In ECCV 2022.

The official Tensorflow 2 implementation of our high quality frame interpolation neural network. We present a unified single-network approach that doesn't use additional pre-trained networks, like optical flow or depth, and yet achieve state-of-the-art results. We use a multi-scale feature extractor that shares the same convolution weights across the scales. Our model is trainable from frame triplets alone.

FILM transforms near-duplicate photos into a slow motion footage that look like it is shot with a video camera.

效果

模型信息

Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：I0
tensor：Float[1, 3, -1, -1]
name：I1
tensor：Float[1, 3, -1, -1]
---------------------------------------------------------------

Outputs
-------------------------
name：merged
tensor：Float[1, -1, -1, -1]
---------------------------------------------------------------

项目

代码

using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using OpenCvSharp;
using OpenCvSharp.Dnn;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
using System.Numerics;
using System.Windows.Forms;

namespace Onnx_Yolov8_Demo
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        string fileFilter = "*.*|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
        string image_path = "";
        string startupPath;
        DateTime dt1 = DateTime.Now;
        DateTime dt2 = DateTime.Now;
        string model_path;
        Mat image;
        Mat result_image;

        SessionOptions options;
        InferenceSession onnx_session;
        Tensor input_tensor;
        Tensor input_tensor2;
        List input_container;
        IDisposableReadOnlyCollection result_infer;
        DisposableNamedOnnxValue[] results_onnxvalue;

        Tensor result_tensors;
        float[] result_array;

        float[] input1_image;
        float[] input2_image;

        int inpWidth;
        int inpHeight;

        private void button1_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = fileFilter;
            if (ofd.ShowDialog() != DialogResult.OK) return;
            pictureBox1.Image = null;
            image_path = ofd.FileName;
            pictureBox1.Image = new Bitmap(image_path);
            textBox1.Text = "";
            image = new Mat(image_path);
            pictureBox2.Image = null;
        }

        void Preprocess(Mat img, ref float[] input_img)
        {
            Mat rgbimg = new Mat();
            Cv2.CvtColor(img, rgbimg, ColorConversionCodes.BGR2RGB);
            int h = rgbimg.Rows;
            int w = rgbimg.Cols;
            int align = 32;
            if (h % align != 0 || w % align != 0)
            {
                int ph = ((h - 1) / align + 1) * align;
                int pw = ((w - 1) / align + 1) * align;

                Cv2.CopyMakeBorder(rgbimg, rgbimg, 0, ph - h, 0, pw - w, BorderTypes.Constant, 0);
            }

            inpHeight = rgbimg.Rows;
            inpWidth = rgbimg.Cols;

            rgbimg.ConvertTo(rgbimg, MatType.CV_32FC3, 1 / 255.0);

            int image_area = rgbimg.Rows * rgbimg.Cols;

            //input_img = new float[3 * image_area];

            input_img = Common.ExtractMat(rgbimg);

        }

        Mat Interpolate(Mat srcimg1, Mat srcimg2)
        {
            int srch = srcimg1.Rows;
            int srcw = srcimg1.Cols;

            Preprocess(srcimg1, ref input1_image);
            Preprocess(srcimg2, ref input2_image);

            // 输入Tensor
            input_tensor = new DenseTensor(input1_image, new[] { 1, 3, inpHeight, inpWidth });
            input_tensor2 = new DenseTensor(input2_image, new[] { 1, 3, inpHeight, inpWidth });

            //将tensor 放入一个输入参数的容器，并指定名称
            input_container.Add(NamedOnnxValue.CreateFromTensor("I0", input_tensor));
            input_container.Add(NamedOnnxValue.CreateFromTensor("I1", input_tensor2));

            //运行 Inference 并获取结果
            result_infer = onnx_session.Run(input_container);

            // 将输出结果转为DisposableNamedOnnxValue数组
            results_onnxvalue = result_infer.ToArray();

            // 读取第一个节点输出并转为Tensor数据
            result_tensors = results_onnxvalue[0].AsTensor();

            int out_h = results_onnxvalue[0].AsTensor().Dimensions[2];
            int out_w = results_onnxvalue[0].AsTensor().Dimensions[3];

            result_array = result_tensors.ToArray();

            for (int i = 0; i < result_array.Length; i++)
            {
                result_array[i] = result_array[i] * 255;

                if (result_array[i] < 0)
                {
                    result_array[i] = 0;
                }
                else if (result_array[i] > 255)
                {
                    result_array[i] = 255;
                }

                result_array[i] = result_array[i] + 0.5f;
            }

            float[] temp_r = new float[out_h * out_w];
            float[] temp_g = new float[out_h * out_w];
            float[] temp_b = new float[out_h * out_w];

            Array.Copy(result_array, temp_r, out_h * out_w);
            Array.Copy(result_array, out_h * out_w, temp_g, 0, out_h * out_w);
            Array.Copy(result_array, out_h * out_w * 2, temp_b, 0, out_h * out_w);

            Mat rmat = new Mat(out_h, out_w, MatType.CV_32F, temp_r);
            Mat gmat = new Mat(out_h, out_w, MatType.CV_32F, temp_g);
            Mat bmat = new Mat(out_h, out_w, MatType.CV_32F, temp_b);

            result_image = new Mat();
            Cv2.Merge(new Mat[] { bmat, gmat, rmat }, result_image);

            result_image.ConvertTo(result_image, MatType.CV_8UC3);

            Mat mid_img = new Mat(result_image, new Rect(0, 0, srcw, srch));

            return mid_img;

        }

        private void button2_Click(object sender, EventArgs e)
        {
            button2.Enabled = false;
            pictureBox2.Image = null;
            textBox1.Text = "正在运行，请稍后……";
            Application.DoEvents();

            dt1 = DateTime.Now;

            List inputs_imgpath = new List() { "test_img/frame07.png", "test_img/frame08.png", "test_img/frame09.png", "test_img/frame10.png", "test_img/frame11.png", "test_img/frame12.png", "test_img/frame13.png", "test_img/frame14.png" };

            int imgnum = inputs_imgpath.Count();

            for (int i = 0; i < imgnum - 1; i++)
            {
                Mat srcimg1 = Cv2.ImRead(inputs_imgpath[i]);
                Mat srcimg2 = Cv2.ImRead(inputs_imgpath[i + 1]);

                Mat mid_img = Interpolate(srcimg1, srcimg2);

                string save_imgpath = "imgs_results/mid" + i + ".jpg";
                Cv2.ImWrite(save_imgpath, mid_img);
            }

            dt2 = DateTime.Now;

            textBox1.Text = "推理耗时:" + (dt2 - dt1).TotalMilliseconds + "ms";
            button2.Enabled = true;
        }

        private void Form1_Load(object sender, EventArgs e)
        {
            model_path = "model/RIFE_HDv3.onnx";

            // 创建输出会话，用于输出模型读取信息
            options = new SessionOptions();
            options.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_INFO;
            options.AppendExecutionProvider_CPU(0);// 设置为CPU上运行

            // 创建推理模型类，读取本地模型文件
            onnx_session = new InferenceSession(model_path, options);//model_path 为onnx模型文件的路径

            // 创建输入容器
            input_container = new List();

            pictureBox1.Image = new Bitmap("test_img/frame11.png");
            pictureBox3.Image = new Bitmap("test_img/frame12.png");

        }

        private void pictureBox1_DoubleClick(object sender, EventArgs e)
        {
            Common.ShowNormalImg(pictureBox1.Image);
        }

        private void pictureBox2_DoubleClick(object sender, EventArgs e)
        {
            Common.ShowNormalImg(pictureBox2.Image);
        }

        SaveFileDialog sdf = new SaveFileDialog();
        private void button3_Click(object sender, EventArgs e)
        {
            if (pictureBox2.Image == null)
            {
                return;
            }
            Bitmap output = new Bitmap(pictureBox2.Image);
            sdf.Title = "保存";
            sdf.Filter = "Images (*.jpg)|*.jpg|Images (*.png)|*.png|Images (*.bmp)|*.bmp|Images (*.emf)|*.emf|Images (*.exif)|*.exif|Images (*.gif)|*.gif|Images (*.ico)|*.ico|Images (*.tiff)|*.tiff|Images (*.wmf)|*.wmf";
            if (sdf.ShowDialog() == DialogResult.OK)
            {
                switch (sdf.FilterIndex)
                {
                    case 1:
                        {
                            output.Save(sdf.FileName, ImageFormat.Jpeg);
                            break;
                        }
                    case 2:
                        {
                            output.Save(sdf.FileName, ImageFormat.Png);
                            break;
                        }
                    case 3:
                        {
                            output.Save(sdf.FileName, ImageFormat.Bmp);
                            break;
                        }
                    case 4:
                        {
                            output.Save(sdf.FileName, ImageFormat.Emf);
                            break;
                        }
                    case 5:
                        {
                            output.Save(sdf.FileName, ImageFormat.Exif);
                            break;
                        }
                    case 6:
                        {
                            output.Save(sdf.FileName, ImageFormat.Gif);
                            break;
                        }
                    case 7:
                        {
                            output.Save(sdf.FileName, ImageFormat.Icon);
                            break;
                        }

                    case 8:
                        {
                            output.Save(sdf.FileName, ImageFormat.Tiff);
                            break;
                        }
                    case 9:
                        {
                            output.Save(sdf.FileName, ImageFormat.Wmf);
                            break;
                        }
                }
                MessageBox.Show("保存成功，位置：" + sdf.FileName);
            }
        }

        private void button4_Click(object sender, EventArgs e)
        {
            button2.Enabled = false;
            pictureBox2.Image = null;
            textBox1.Text = "正在运行，请稍后……";
            Application.DoEvents();

            dt1 = DateTime.Now;

            Mat srcimg1 = Cv2.ImRead("test_img/frame11.png");
            Mat srcimg2 = Cv2.ImRead("test_img/frame12.png");

            Mat mid_img = Interpolate(srcimg1, srcimg2);

            dt2 = DateTime.Now;

            pictureBox2.Image = new Bitmap(mid_img.ToMemoryStream());

            textBox1.Text = "推理耗时:" + (dt2 - dt1).TotalMilliseconds + "ms";
            button2.Enabled = true;
        }
    }
}

  1. using Microsoft.ML.OnnxRuntime;
  2. using Microsoft.ML.OnnxRuntime.Tensors;
  3. using OpenCvSharp;
  4. using OpenCvSharp.Dnn;
  5. using System;
  6. using System.Collections.Generic;
  7. using System.Drawing;
  8. using System.Drawing.Imaging;
  9. using System.Linq;
  10. using System.Numerics;
  11. using System.Windows.Forms;
  12.  
  13. namespace Onnx_Yolov8_Demo
  14. {
  15.     public partial class Form1 : Form
  16.     {
  17.         public Form1()
  18.         {
  19.             InitializeComponent();
  20.         }
  21.  
  22.         string fileFilter = "*.*|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
  23.         string image_path = "";
  24.         string startupPath;
  25.         DateTime dt1 = DateTime.Now;
  26.         DateTime dt2 = DateTime.Now;
  27.         string model_path;
  28.         Mat image;
  29.         Mat result_image;
  30.  
  31.         SessionOptions options;
  32.         InferenceSession onnx_session;
  33.         Tensor<float> input_tensor;
  34.         Tensor<float> input_tensor2;
  35.         List<NamedOnnxValue> input_container;
  36.         IDisposableReadOnlyCollection<DisposableNamedOnnxValue> result_infer;
  37.         DisposableNamedOnnxValue[] results_onnxvalue;
  38.  
  39.         Tensor<float> result_tensors;
  40.         float[] result_array;
  41.  
  42.         float[] input1_image;
  43.         float[] input2_image;
  44.  
  45.         int inpWidth;
  46.         int inpHeight;
  47.  
  48.         private void button1_Click(object sender, EventArgs e)
  49.         {
  50.             OpenFileDialog ofd = new OpenFileDialog();
  51.             ofd.Filter = fileFilter;
  52.             if (ofd.ShowDialog() != DialogResult.OK) return;
  53.             pictureBox1.Image = null;
  54.             image_path = ofd.FileName;
  55.             pictureBox1.Image = new Bitmap(image_path);
  56.             textBox1.Text = "";
  57.             image = new Mat(image_path);
  58.             pictureBox2.Image = null;
  59.         }
  60.  
  61.         void Preprocess(Mat img, ref float[] input_img)
  62.         {
  63.             Mat rgbimg = new Mat();
  64.             Cv2.CvtColor(img, rgbimg, ColorConversionCodes.BGR2RGB);
  65.             int h = rgbimg.Rows;
  66.             int w = rgbimg.Cols;
  67.             int align = 32;
  68.             if (h % align != 0 || w % align != 0)
  69.             {
  70.                 int ph = ((h - 1) / align + 1) * align;
  71.                 int pw = ((w - 1) / align + 1) * align;
  72.  
  73.                 Cv2.CopyMakeBorder(rgbimg, rgbimg, 0, ph - h, 0, pw - w, BorderTypes.Constant, 0);
  74.             }
  75.  
  76.             inpHeight = rgbimg.Rows;
  77.             inpWidth = rgbimg.Cols;
  78.  
  79.             rgbimg.ConvertTo(rgbimg, MatType.CV_32FC3, 1 / 255.0);
  80.  
  81.             int image_area = rgbimg.Rows * rgbimg.Cols;
  82.  
  83.             //input_img = new float[3 * image_area];
  84.  
  85.             input_img = Common.ExtractMat(rgbimg);
  86.  
  87.         }
  88.  
  89.         Mat Interpolate(Mat srcimg1, Mat srcimg2)
  90.         {
  91.             int srch = srcimg1.Rows;
  92.             int srcw = srcimg1.Cols;
  93.  
  94.             Preprocess(srcimg1, ref input1_image);
  95.             Preprocess(srcimg2, ref input2_image);
  96.  
  97.             // 输入Tensor
  98.             input_tensor = new DenseTensor<float>(input1_image, new[] { 1, 3, inpHeight, inpWidth });
  99.             input_tensor2 = new DenseTensor<float>(input2_image, new[] { 1, 3, inpHeight, inpWidth });
  100.  
  101.             //将tensor 放入一个输入参数的容器，并指定名称
  102.             input_container.Add(NamedOnnxValue.CreateFromTensor("I0", input_tensor));
  103.             input_container.Add(NamedOnnxValue.CreateFromTensor("I1", input_tensor2));
  104.  
  105.             //运行 Inference 并获取结果
  106.             result_infer = onnx_session.Run(input_container);
  107.  
  108.             // 将输出结果转为DisposableNamedOnnxValue数组
  109.             results_onnxvalue = result_infer.ToArray();
  110.  
  111.             // 读取第一个节点输出并转为Tensor数据
  112.             result_tensors = results_onnxvalue[0].AsTensor<float>();
  113.  
  114.             int out_h = results_onnxvalue[0].AsTensor<float>().Dimensions[2];
  115.             int out_w = results_onnxvalue[0].AsTensor<float>().Dimensions[3];
  116.  
  117.             result_array = result_tensors.ToArray();
  118.  
  119.             for (int i = 0; i < result_array.Length; i++)
  120.             {
  121.                 result_array[i] = result_array[i] * 255;
  122.  
  123.                 if (result_array[i] < 0)
  124.                 {
  125.                     result_array[i] = 0;
  126.                 }
  127.                 else if (result_array[i] > 255)
  128.                 {
  129.                     result_array[i] = 255;
  130.                 }
  131.  
  132.                 result_array[i] = result_array[i] + 0.5f;
  133.             }
  134.  
  135.             float[] temp_r = new float[out_h * out_w];
  136.             float[] temp_g = new float[out_h * out_w];
  137.             float[] temp_b = new float[out_h * out_w];
  138.  
  139.             Array.Copy(result_array, temp_r, out_h * out_w);
  140.             Array.Copy(result_array, out_h * out_w, temp_g, 0, out_h * out_w);
  141.             Array.Copy(result_array, out_h * out_w * 2, temp_b, 0, out_h * out_w);
  142.  
  143.             Mat rmat = new Mat(out_h, out_w, MatType.CV_32F, temp_r);
  144.             Mat gmat = new Mat(out_h, out_w, MatType.CV_32F, temp_g);
  145.             Mat bmat = new Mat(out_h, out_w, MatType.CV_32F, temp_b);
  146.  
  147.             result_image = new Mat();
  148.             Cv2.Merge(new Mat[] { bmat, gmat, rmat }, result_image);
  149.  
  150.             result_image.ConvertTo(result_image, MatType.CV_8UC3);
  151.  
  152.             Mat mid_img = new Mat(result_image, new Rect(0, 0, srcw, srch));
  153.  
  154.             return mid_img;
  155.  
  156.         }
  157.  
  158.         private void button2_Click(object sender, EventArgs e)
  159.         {
  160.             button2.Enabled = false;
  161.             pictureBox2.Image = null;
  162.             textBox1.Text = "正在运行，请稍后……";
  163.             Application.DoEvents();
  164.  
  165.             dt1 = DateTime.Now;
  166.  
  167.             List<String> inputs_imgpath = new List<String>() { "test_img/frame07.png", "test_img/frame08.png", "test_img/frame09.png", "test_img/frame10.png", "test_img/frame11.png", "test_img/frame12.png", "test_img/frame13.png", "test_img/frame14.png" };
  168.  
  169.             int imgnum = inputs_imgpath.Count();
  170.  
  171.             for (int i = 0; i < imgnum - 1; i++)
  172.             {
  173.                 Mat srcimg1 = Cv2.ImRead(inputs_imgpath[i]);
  174.                 Mat srcimg2 = Cv2.ImRead(inputs_imgpath[i + 1]);
  175.  
  176.                 Mat mid_img = Interpolate(srcimg1, srcimg2);
  177.  
  178.                 string save_imgpath = "imgs_results/mid" + i + ".jpg";
  179.                 Cv2.ImWrite(save_imgpath, mid_img);
  180.             }
  181.  
  182.             dt2 = DateTime.Now;
  183.  
  184.             textBox1.Text = "推理耗时:" + (dt2 - dt1).TotalMilliseconds + "ms";
  185.             button2.Enabled = true;
  186.         }
  187.  
  188.         private void Form1_Load(object sender, EventArgs e)
  189.         {
  190.             model_path = "model/RIFE_HDv3.onnx";
  191.  
  192.             // 创建输出会话，用于输出模型读取信息
  193.             options = new SessionOptions();
  194.             options.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_INFO;
  195.             options.AppendExecutionProvider_CPU(0);// 设置为CPU上运行
  196.  
  197.             // 创建推理模型类，读取本地模型文件
  198.             onnx_session = new InferenceSession(model_path, options);//model_path 为onnx模型文件的路径
  199.  
  200.             // 创建输入容器
  201.             input_container = new List<NamedOnnxValue>();
  202.  
  203.             pictureBox1.Image = new Bitmap("test_img/frame11.png");
  204.             pictureBox3.Image = new Bitmap("test_img/frame12.png");
  205.  
  206.         }
  207.  
  208.         private void pictureBox1_DoubleClick(object sender, EventArgs e)
  209.         {
  210.             Common.ShowNormalImg(pictureBox1.Image);
  211.         }
  212.  
  213.         private void pictureBox2_DoubleClick(object sender, EventArgs e)
  214.         {
  215.             Common.ShowNormalImg(pictureBox2.Image);
  216.         }
  217.  
  218.         SaveFileDialog sdf = new SaveFileDialog();
  219.         private void button3_Click(object sender, EventArgs e)
  220.         {
  221.             if (pictureBox2.Image == null)
  222.             {
  223.                 return;
  224.             }
  225.             Bitmap output = new Bitmap(pictureBox2.Image);
  226.             sdf.Title = "保存";
  227.             sdf.Filter = "Images (*.jpg)|*.jpg|Images (*.png)|*.png|Images (*.bmp)|*.bmp|Images (*.emf)|*.emf|Images (*.exif)|*.exif|Images (*.gif)|*.gif|Images (*.ico)|*.ico|Images (*.tiff)|*.tiff|Images (*.wmf)|*.wmf";
  228.             if (sdf.ShowDialog() == DialogResult.OK)
  229.             {
  230.                 switch (sdf.FilterIndex)
  231.                 {
  232.                     case 1:
  233.                         {
  234.                             output.Save(sdf.FileName, ImageFormat.Jpeg);
  235.                             break;
  236.                         }
  237.                     case 2:
  238.                         {
  239.                             output.Save(sdf.FileName, ImageFormat.Png);
  240.                             break;
  241.                         }
  242.                     case 3:
  243.                         {
  244.                             output.Save(sdf.FileName, ImageFormat.Bmp);
  245.                             break;
  246.                         }
  247.                     case 4:
  248.                         {
  249.                             output.Save(sdf.FileName, ImageFormat.Emf);
  250.                             break;
  251.                         }
  252.                     case 5:
  253.                         {
  254.                             output.Save(sdf.FileName, ImageFormat.Exif);
  255.                             break;
  256.                         }
  257.                     case 6:
  258.                         {
  259.                             output.Save(sdf.FileName, ImageFormat.Gif);
  260.                             break;
  261.                         }
  262.                     case 7:
  263.                         {
  264.                             output.Save(sdf.FileName, ImageFormat.Icon);
  265.                             break;
  266.                         }
  267.  
  268.                     case 8:
  269.                         {
  270.                             output.Save(sdf.FileName, ImageFormat.Tiff);
  271.                             break;
  272.                         }
  273.                     case 9:
  274.                         {
  275.                             output.Save(sdf.FileName, ImageFormat.Wmf);
  276.                             break;
  277.                         }
  278.                 }
  279.                 MessageBox.Show("保存成功，位置：" + sdf.FileName);
  280.             }
  281.         }
  282.  
  283.         private void button4_Click(object sender, EventArgs e)
  284.         {
  285.             button2.Enabled = false;
  286.             pictureBox2.Image = null;
  287.             textBox1.Text = "正在运行，请稍后……";
  288.             Application.DoEvents();
  289.  
  290.             dt1 = DateTime.Now;
  291.  
  292.             Mat srcimg1 = Cv2.ImRead("test_img/frame11.png");
  293.             Mat srcimg2 = Cv2.ImRead("test_img/frame12.png");
  294.  
  295.             Mat mid_img = Interpolate(srcimg1, srcimg2);
  296.  
  297.             dt2 = DateTime.Now;
  298.  
  299.             pictureBox2.Image = new Bitmap(mid_img.ToMemoryStream());
  300.  
  301.             textBox1.Text = "推理耗时:" + (dt2 - dt1).TotalMilliseconds + "ms";
  302.             button2.Enabled = true;
  303.         }
  304.     }
  305. }

下载

源码下载

天天代码码天天
微信公众号
.NET 人工智能实践
注：本文转载自blog.csdn.net的天天代码码天天的文章"https://lw112190.blog.csdn.net/article/details/136232915"。版权归原作者所有，此博客不拥有其著作权，亦不承担相应法律责任。如有侵权，请联系我们删除。
复制链接
相关推荐
发表评论

评论记录：

未查询到任何数据！