C# OnnxRuntime SAM2

目录

介绍

效果

模型信息

sam2_hiera_small_encoder.onnx

sam2_hiera_small_decoder.onnx

项目

​编辑

代码

下载

其他

介绍

官网地址：https://github.com/facebookresearch/sam2

Segment Anything Model 2 (SAM 2) is a foundation model towards solving promptable visual segmentation in images and videos. We extend SAM to video by considering images as a video with a single frame. The model design is a simple transformer architecture with streaming memory for real-time video processing. We build a model-in-the-loop data engine, which improves model and data via user interaction, to collect our SA-V dataset, the largest video segmentation dataset to date. SAM 2 trained on our data provides strong performance across a wide range of tasks and visual domains.

效果

模型信息

sam2_hiera_small_encoder.onnx

Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：image
tensor：Float[1, 3, 1024, 1024]
---------------------------------------------------------------

Outputs
-------------------------
name：high_res_feats_0
tensor：Float[1, 32, 256, 256]
name：high_res_feats_1
tensor：Float[1, 64, 128, 128]
name：image_embed
tensor：Float[1, 256, 64, 64]
---------------------------------------------------------------

sam2_hiera_small_decoder.onnx

Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：image_embed
tensor：Float[1, 256, 64, 64]
name：high_res_feats_0
tensor：Float[1, 32, 256, 256]
name：high_res_feats_1
tensor：Float[1, 64, 128, 128]
name：point_coords
tensor：Float[-1, -1, 2]
name：point_labels
tensor：Float[-1, -1]
name：mask_input
tensor：Float[-1, 1, 256, 256]
name：has_mask_input
tensor：Float[-1]
name：orig_im_size
tensor：Int32[2]
---------------------------------------------------------------

Outputs
-------------------------
name：masks
tensor：Float[-1, -1, -1, -1]
name：iou_predictions
tensor：Float[-1, -1]
---------------------------------------------------------------

项目

代码

using OpenCvSharp;
using OpenCvSharp.Extensions;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Reflection;
using System.Windows.Forms;

namespace Onnx_Demo
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        string fileFilter = "*.*|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
        string image_path = "";
        DateTime dt1 = DateTime.Now;
        DateTime dt2 = DateTime.Now;
        Mat image;

        private void button1_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = fileFilter;
            if (ofd.ShowDialog() != DialogResult.OK) return;
            pictureBox1.Image = null;
            image_path = ofd.FileName;
            pictureBox1.Image = new Bitmap(image_path);
            textBox1.Text = "";
            image = new Mat(image_path);
            pictureBox2.Image = null;
            pictureBox3.Image = null;

            sam.SetImage(image_path);
            image = new Mat(image_path);
            pictureBox1.Image = new Bitmap(image_path);
        }

        private void button2_Click(object sender, EventArgs e)
        {
            pictureBox3.Image = null;
            button2.Enabled = false;
            Application.DoEvents();
            List masks_list = sam.GetPointMask(roi);
            if (masks_list.Count > 0)
            {
                int MaxInde = 0;
                float maxiou_pred = masks_list[0].iou_pred;
                for (int i = 0; i < masks_list.Count; i++)
                {
                    float temp = masks_list[i].iou_pred;
                    if (temp > maxiou_pred)
                    {
                        MaxInde = i;
                    }
                }
                pictureBox3.Image = BitmapConverter.ToBitmap(masks_list[MaxInde].mask);
            }
            button2.Enabled = true;
        }

        SamInferenceSession sam;

        private void Form1_Load(object sender, EventArgs e)
        {
            string encoderPath = "model/sam2_hiera_small_encoder.onnx";
            string decoderPath = "model/sam2_hiera_small_decoder.onnx";

            sam = new SamInferenceSession(encoderPath, decoderPath);
            sam.Initialize();

            image_path = "test_img/1.jpg";

            sam.SetImage(image_path);
            image = new Mat(image_path);
            pictureBox1.Image = new Bitmap(image_path);

        }

        private void pictureBox1_DoubleClick(object sender, EventArgs e)
        {
            Common.ShowNormalImg(pictureBox1.Image);
        }

        private void pictureBox2_DoubleClick(object sender, EventArgs e)
        {
            Common.ShowNormalImg(pictureBox2.Image);
        }

        SaveFileDialog sdf = new SaveFileDialog();
        private void button3_Click(object sender, EventArgs e)
        {
            if (pictureBox2.Image == null)
            {
                return;
            }
            Bitmap output = new Bitmap(pictureBox2.Image);
            sdf.Title = "保存";
            sdf.Filter = "Images (*.jpg)|*.jpg|Images (*.png)|*.png|Images (*.bmp)|*.bmp|Images (*.emf)|*.emf|Images (*.exif)|*.exif|Images (*.gif)|*.gif|Images (*.ico)|*.ico|Images (*.tiff)|*.tiff|Images (*.wmf)|*.wmf";
            if (sdf.ShowDialog() == DialogResult.OK)
            {
                switch (sdf.FilterIndex)
                {
                    case 1:
                        {
                            output.Save(sdf.FileName, ImageFormat.Jpeg);
                            break;
                        }
                    case 2:
                        {
                            output.Save(sdf.FileName, ImageFormat.Png);
                            break;
                        }
                    case 3:
                        {
                            output.Save(sdf.FileName, ImageFormat.Bmp);
                            break;
                        }
                    case 4:
                        {
                            output.Save(sdf.FileName, ImageFormat.Emf);
                            break;
                        }
                    case 5:
                        {
                            output.Save(sdf.FileName, ImageFormat.Exif);
                            break;
                        }
                    case 6:
                        {
                            output.Save(sdf.FileName, ImageFormat.Gif);
                            break;
                        }
                    case 7:
                        {
                            output.Save(sdf.FileName, ImageFormat.Icon);
                            break;
                        }

                    case 8:
                        {
                            output.Save(sdf.FileName, ImageFormat.Tiff);
                            break;
                        }
                    case 9:
                        {
                            output.Save(sdf.FileName, ImageFormat.Wmf);
                            break;
                        }
                }
                MessageBox.Show("保存成功，位置：" + sdf.FileName);
            }
        }

        bool m_mouseDown = false;
        bool m_mouseMove = false;

        System.Drawing.Point startPoint = new System.Drawing.Point();
        System.Drawing.Point endPoint = new System.Drawing.Point();

        Mat currentFrame = new Mat();
        Rect roi = new Rect();

        private void pictureBox1_MouseMove(object sender, MouseEventArgs e)
        {
            if (pictureBox1.Image == null)
                return;
            if (!m_mouseDown) return;

            m_mouseMove = true;
            endPoint = e.Location;

            pictureBox1.Invalidate();

        }

        private void pictureBox1_Paint(object sender, PaintEventArgs e)
        {
            if (!m_mouseDown || !m_mouseMove)
                return;
            Graphics g = e.Graphics;
            Pen p = new Pen(Color.Blue, 2);
            Rectangle rect = new Rectangle(startPoint.X, startPoint.Y, (endPoint.X - startPoint.X), (endPoint.Y - startPoint.Y));
            g.DrawRectangle(p, rect);
        }

        private void pictureBox1_MouseUp(object sender, MouseEventArgs e)
        {
            if (!m_mouseDown || !m_mouseMove)
                return;
            m_mouseDown = false;
            m_mouseMove = false;

            System.Drawing.Point image_startPoint = ConvertCooridinate(startPoint);
            System.Drawing.Point image_endPoint = ConvertCooridinate(endPoint);
            if (image_startPoint.X < 0)
                image_startPoint.X = 0;
            if (image_startPoint.Y < 0)
                image_startPoint.Y = 0;
            if (image_endPoint.X < 0)
                image_endPoint.X = 0;
            if (image_endPoint.Y < 0)
                image_endPoint.Y = 0;
            if (image_startPoint.X > image.Cols)
                image_startPoint.X = image.Cols;
            if (image_startPoint.Y > image.Rows)
                image_startPoint.Y = image.Rows;
            if (image_endPoint.X > image.Cols)
                image_endPoint.X = image.Cols;
            if (image_endPoint.Y > image.Rows)
                image_endPoint.Y = image.Rows;

            label1.Text = String.Format("ROI:({0},{1})-({2},{3})", image_startPoint.X, image_startPoint.Y, image_endPoint.X, image_endPoint.Y);
            int w = (image_endPoint.X - image_startPoint.X);
            int h = (image_endPoint.Y - image_startPoint.Y);
            if (w > 10 && h > 10)
            {
                roi = new Rect(image_startPoint.X, image_startPoint.Y, w, h);
                //Console.WriteLine(String.Format("ROI:({0},{1})-({2},{3})", image_startPoint.X, image_startPoint.Y, image_endPoint.X, image_endPoint.Y));
                //test
                //OpenCvSharp.Point pointinfo = new OpenCvSharp.Point(910, 641);
                //roi = new Rect(pointinfo.X - 160, pointinfo.Y - 430, 380, 940);
                Mat roi_mat = image[roi];
                pictureBox2.Image = BitmapConverter.ToBitmap(roi_mat);
            }
            //pictureBox1.Invalidate();

        }

        private void pictureBox1_MouseDown(object sender, MouseEventArgs e)
        {
            if (pictureBox1.Image == null)
                return;
            m_mouseDown = true;
            startPoint = e.Location;
        }

        private System.Drawing.Point ConvertCooridinate(System.Drawing.Point point)
        {
            System.Reflection.PropertyInfo rectangleProperty = this.pictureBox1.GetType().GetProperty("ImageRectangle", BindingFlags.Instance | BindingFlags.NonPublic);
            Rectangle pictureBox = (Rectangle)rectangleProperty.GetValue(this.pictureBox1, null);

            int zoomedWidth = pictureBox.Width;
            int zoomedHeight = pictureBox.Height;

            int imageWidth = pictureBox1.Image.Width;
            int imageHeight = pictureBox1.Image.Height;

            double zoomRatex = (double)(zoomedWidth) / (double)(imageWidth);
            double zoomRatey = (double)(zoomedHeight) / (double)(imageHeight);
            int black_left_width = (zoomedWidth == this.pictureBox1.Width) ? 0 : (this.pictureBox1.Width - zoomedWidth) / 2;
            int black_top_height = (zoomedHeight == this.pictureBox1.Height) ? 0 : (this.pictureBox1.Height - zoomedHeight) / 2;

            int zoomedX = point.X - black_left_width;
            int zoomedY = point.Y - black_top_height;

            System.Drawing.Point outPoint = new System.Drawing.Point();
            outPoint.X = (int)((double)zoomedX / zoomRatex);
            outPoint.Y = (int)((double)zoomedY / zoomRatey);

            return outPoint;
        }

    }
}
 

  1. using OpenCvSharp;
  2. using OpenCvSharp.Extensions;
  3. using System;
  4. using System.Collections.Generic;
  5. using System.Drawing;
  6. using System.Drawing.Imaging;
  7. using System.Reflection;
  8. using System.Windows.Forms;
  9.  
  10. namespace Onnx_Demo
  11. {
  12.     public partial class Form1 : Form
  13.     {
  14.         public Form1()
  15.         {
  16.             InitializeComponent();
  17.         }
  18.  
  19.         string fileFilter = "*.*|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
  20.         string image_path = "";
  21.         DateTime dt1 = DateTime.Now;
  22.         DateTime dt2 = DateTime.Now;
  23.         Mat image;
  24.  
  25.         private void button1_Click(object sender, EventArgs e)
  26.         {
  27.             OpenFileDialog ofd = new OpenFileDialog();
  28.             ofd.Filter = fileFilter;
  29.             if (ofd.ShowDialog() != DialogResult.OK) return;
  30.             pictureBox1.Image = null;
  31.             image_path = ofd.FileName;
  32.             pictureBox1.Image = new Bitmap(image_path);
  33.             textBox1.Text = "";
  34.             image = new Mat(image_path);
  35.             pictureBox2.Image = null;
  36.             pictureBox3.Image = null;
  37.  
  38.             sam.SetImage(image_path);
  39.             image = new Mat(image_path);
  40.             pictureBox1.Image = new Bitmap(image_path);
  41.         }
  42.  
  43.         private void button2_Click(object sender, EventArgs e)
  44.         {
  45.             pictureBox3.Image = null;
  46.             button2.Enabled = false;
  47.             Application.DoEvents();
  48.             List<MatInfo> masks_list = sam.GetPointMask(roi);
  49.             if (masks_list.Count > 0)
  50.             {
  51.                 int MaxInde = 0;
  52.                 float maxiou_pred = masks_list[0].iou_pred;
  53.                 for (int i = 0; i < masks_list.Count; i++)
  54.                 {
  55.                     float temp = masks_list[i].iou_pred;
  56.                     if (temp > maxiou_pred)
  57.                     {
  58.                         MaxInde = i;
  59.                     }
  60.                 }
  61.                 pictureBox3.Image = BitmapConverter.ToBitmap(masks_list[MaxInde].mask);
  62.             }
  63.             button2.Enabled = true;
  64.         }
  65.  
  66.         SamInferenceSession sam;
  67.  
  68.         private void Form1_Load(object sender, EventArgs e)
  69.         {
  70.             string encoderPath = "model/sam2_hiera_small_encoder.onnx";
  71.             string decoderPath = "model/sam2_hiera_small_decoder.onnx";
  72.  
  73.             sam = new SamInferenceSession(encoderPath, decoderPath);
  74.             sam.Initialize();
  75.  
  76.             image_path = "test_img/1.jpg";
  77.  
  78.             sam.SetImage(image_path);
  79.             image = new Mat(image_path);
  80.             pictureBox1.Image = new Bitmap(image_path);
  81.  
  82.         }
  83.  
  84.         private void pictureBox1_DoubleClick(object sender, EventArgs e)
  85.         {
  86.             Common.ShowNormalImg(pictureBox1.Image);
  87.         }
  88.  
  89.         private void pictureBox2_DoubleClick(object sender, EventArgs e)
  90.         {
  91.             Common.ShowNormalImg(pictureBox2.Image);
  92.         }
  93.  
  94.         SaveFileDialog sdf = new SaveFileDialog();
  95.         private void button3_Click(object sender, EventArgs e)
  96.         {
  97.             if (pictureBox2.Image == null)
  98.             {
  99.                 return;
  100.             }
  101.             Bitmap output = new Bitmap(pictureBox2.Image);
  102.             sdf.Title = "保存";
  103.             sdf.Filter = "Images (*.jpg)|*.jpg|Images (*.png)|*.png|Images (*.bmp)|*.bmp|Images (*.emf)|*.emf|Images (*.exif)|*.exif|Images (*.gif)|*.gif|Images (*.ico)|*.ico|Images (*.tiff)|*.tiff|Images (*.wmf)|*.wmf";
  104.             if (sdf.ShowDialog() == DialogResult.OK)
  105.             {
  106.                 switch (sdf.FilterIndex)
  107.                 {
  108.                     case 1:
  109.                         {
  110.                             output.Save(sdf.FileName, ImageFormat.Jpeg);
  111.                             break;
  112.                         }
  113.                     case 2:
  114.                         {
  115.                             output.Save(sdf.FileName, ImageFormat.Png);
  116.                             break;
  117.                         }
  118.                     case 3:
  119.                         {
  120.                             output.Save(sdf.FileName, ImageFormat.Bmp);
  121.                             break;
  122.                         }
  123.                     case 4:
  124.                         {
  125.                             output.Save(sdf.FileName, ImageFormat.Emf);
  126.                             break;
  127.                         }
  128.                     case 5:
  129.                         {
  130.                             output.Save(sdf.FileName, ImageFormat.Exif);
  131.                             break;
  132.                         }
  133.                     case 6:
  134.                         {
  135.                             output.Save(sdf.FileName, ImageFormat.Gif);
  136.                             break;
  137.                         }
  138.                     case 7:
  139.                         {
  140.                             output.Save(sdf.FileName, ImageFormat.Icon);
  141.                             break;
  142.                         }
  143.  
  144.                     case 8:
  145.                         {
  146.                             output.Save(sdf.FileName, ImageFormat.Tiff);
  147.                             break;
  148.                         }
  149.                     case 9:
  150.                         {
  151.                             output.Save(sdf.FileName, ImageFormat.Wmf);
  152.                             break;
  153.                         }
  154.                 }
  155.                 MessageBox.Show("保存成功，位置：" + sdf.FileName);
  156.             }
  157.         }
  158.  
  159.         bool m_mouseDown = false;
  160.         bool m_mouseMove = false;
  161.  
  162.         System.Drawing.Point startPoint = new System.Drawing.Point();
  163.         System.Drawing.Point endPoint = new System.Drawing.Point();
  164.  
  165.         Mat currentFrame = new Mat();
  166.         Rect roi = new Rect();
  167.  
  168.         private void pictureBox1_MouseMove(object sender, MouseEventArgs e)
  169.         {
  170.             if (pictureBox1.Image == null)
  171.                 return;
  172.             if (!m_mouseDown) return;
  173.  
  174.             m_mouseMove = true;
  175.             endPoint = e.Location;
  176.  
  177.             pictureBox1.Invalidate();
  178.  
  179.         }
  180.  
  181.         private void pictureBox1_Paint(object sender, PaintEventArgs e)
  182.         {
  183.             if (!m_mouseDown || !m_mouseMove)
  184.                 return;
  185.             Graphics g = e.Graphics;
  186.             Pen p = new Pen(Color.Blue, 2);
  187.             Rectangle rect = new Rectangle(startPoint.X, startPoint.Y, (endPoint.X - startPoint.X), (endPoint.Y - startPoint.Y));
  188.             g.DrawRectangle(p, rect);
  189.         }
  190.  
  191.         private void pictureBox1_MouseUp(object sender, MouseEventArgs e)
  192.         {
  193.             if (!m_mouseDown || !m_mouseMove)
  194.                 return;
  195.             m_mouseDown = false;
  196.             m_mouseMove = false;
  197.  
  198.             System.Drawing.Point image_startPoint = ConvertCooridinate(startPoint);
  199.             System.Drawing.Point image_endPoint = ConvertCooridinate(endPoint);
  200.             if (image_startPoint.X < 0)
  201.                 image_startPoint.X = 0;
  202.             if (image_startPoint.Y < 0)
  203.                 image_startPoint.Y = 0;
  204.             if (image_endPoint.X < 0)
  205.                 image_endPoint.X = 0;
  206.             if (image_endPoint.Y < 0)
  207.                 image_endPoint.Y = 0;
  208.             if (image_startPoint.X > image.Cols)
  209.                 image_startPoint.X = image.Cols;
  210.             if (image_startPoint.Y > image.Rows)
  211.                 image_startPoint.Y = image.Rows;
  212.             if (image_endPoint.X > image.Cols)
  213.                 image_endPoint.X = image.Cols;
  214.             if (image_endPoint.Y > image.Rows)
  215.                 image_endPoint.Y = image.Rows;
  216.  
  217.             label1.Text = String.Format("ROI:({0},{1})-({2},{3})", image_startPoint.X, image_startPoint.Y, image_endPoint.X, image_endPoint.Y);
  218.             int w = (image_endPoint.X - image_startPoint.X);
  219.             int h = (image_endPoint.Y - image_startPoint.Y);
  220.             if (w > 10 && h > 10)
  221.             {
  222.                 roi = new Rect(image_startPoint.X, image_startPoint.Y, w, h);
  223.                 //Console.WriteLine(String.Format("ROI:({0},{1})-({2},{3})", image_startPoint.X, image_startPoint.Y, image_endPoint.X, image_endPoint.Y));
  224.                 //test
  225.                 //OpenCvSharp.Point pointinfo = new OpenCvSharp.Point(910, 641);
  226.                 //roi = new Rect(pointinfo.X - 160, pointinfo.Y - 430, 380, 940);
  227.                 Mat roi_mat = image[roi];
  228.                 pictureBox2.Image = BitmapConverter.ToBitmap(roi_mat);
  229.             }
  230.             //pictureBox1.Invalidate();
  231.  
  232.         }
  233.  
  234.         private void pictureBox1_MouseDown(object sender, MouseEventArgs e)
  235.         {
  236.             if (pictureBox1.Image == null)
  237.                 return;
  238.             m_mouseDown = true;
  239.             startPoint = e.Location;
  240.         }
  241.  
  242.         private System.Drawing.Point ConvertCooridinate(System.Drawing.Point point)
  243.         {
  244.             System.Reflection.PropertyInfo rectangleProperty = this.pictureBox1.GetType().GetProperty("ImageRectangle", BindingFlags.Instance | BindingFlags.NonPublic);
  245.             Rectangle pictureBox = (Rectangle)rectangleProperty.GetValue(this.pictureBox1, null);
  246.  
  247.             int zoomedWidth = pictureBox.Width;
  248.             int zoomedHeight = pictureBox.Height;
  249.  
  250.             int imageWidth = pictureBox1.Image.Width;
  251.             int imageHeight = pictureBox1.Image.Height;
  252.  
  253.             double zoomRatex = (double)(zoomedWidth) / (double)(imageWidth);
  254.             double zoomRatey = (double)(zoomedHeight) / (double)(imageHeight);
  255.             int black_left_width = (zoomedWidth == this.pictureBox1.Width) ? 0 : (this.pictureBox1.Width - zoomedWidth) / 2;
  256.             int black_top_height = (zoomedHeight == this.pictureBox1.Height) ? 0 : (this.pictureBox1.Height - zoomedHeight) / 2;
  257.  
  258.             int zoomedX = point.X - black_left_width;
  259.             int zoomedY = point.Y - black_top_height;
  260.  
  261.             System.Drawing.Point outPoint = new System.Drawing.Point();
  262.             outPoint.X = (int)((double)zoomedX / zoomRatex);
  263.             outPoint.Y = (int)((double)zoomedY / zoomRatey);
  264.  
  265.             return outPoint;
  266.         }
  267.  
  268.     }
  269. }

下载

源码下载

其他

C# OnnxRuntime SAM

天天代码码天天
微信公众号
.NET 人工智能实践
注：本文转载自blog.csdn.net的天天代码码天天的文章"https://lw112190.blog.csdn.net/article/details/143889764"。版权归原作者所有，此博客不拥有其著作权，亦不承担相应法律责任。如有侵权，请联系我们删除。
复制链接
相关推荐
发表评论

评论记录：

未查询到任何数据！