C# OnnxRuntime Gaze-LLE 凝视目标估计，通过利用冻结的DINOv2编码器的特征来简化注视目标估计，预测一个人在场景中看的位置。

目录

说明

效果

​编辑模型信息

det_face.onnx

gazelle_dinov2_vitl14_inout_1x3x448x448_1xNx4.onnx

项目

代码

下载

参考

说明

github地址：https://github.com/fkryan/gazelle

This is the official implementation for Gaze-LLE, a transformer approach for estimating gaze targets that leverages the power of pretrained visual foundation models. Gaze-LLE provides a streamlined gaze architecture that learns only a lightweight gaze decoder on top of a frozen, pretrained visual encoder (DINOv2). Gaze-LLE learns 1-2 orders of magnitude fewer parameters than prior works and doesn't require any extra input modalities like depth and pose!

效果

模型信息

det_face.onnx

Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：input.1
tensor：Float[1, 3, -1, -1]
---------------------------------------------------------------

Outputs
-------------------------
name：448
tensor：Float[12800, 1]
name：471
tensor：Float[3200, 1]
name：494
tensor：Float[800, 1]
name：451
tensor：Float[12800, 4]
name：474
tensor：Float[3200, 4]
name：497
tensor：Float[800, 4]
name：454
tensor：Float[12800, 10]
name：477
tensor：Float[3200, 10]
name：500
tensor：Float[800, 10]
---------------------------------------------------------------

gazelle_dinov2_vitl14_inout_1x3x448x448_1xNx4.onnx

Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：image_bgr
tensor：Float[1, 3, 448, 448]
name：bboxes_x1y1x2y2
tensor：Float[1, -1, 4]
---------------------------------------------------------------

Outputs
-------------------------
name：heatmap
tensor：Float[-1, 64, 64]
name：inout
tensor：Float[-1]
---------------------------------------------------------------

项目

代码

using OpenCvSharp;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Drawing.Imaging;
using System.Windows.Forms;

namespace Onnx_Demo
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        string fileFilter = "*.*|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
        string image_path = "";
        DateTime dt1 = DateTime.Now;
        DateTime dt2 = DateTime.Now;

        Mat image;
        Mat result_image;

        FaceDet face_det;
        GazeLLE gazelle;

        private void button1_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = fileFilter;
            if (ofd.ShowDialog() != DialogResult.OK) return;
            pictureBox1.Image = null;
            image_path = ofd.FileName;
            pictureBox1.Image = new Bitmap(image_path);
            textBox1.Text = "";
            image = new Mat(image_path);
            pictureBox2.Image = null;
        }

        private void button2_Click(object sender, EventArgs e)
        {
            if (image_path == "")
            {
                return;
            }

            button2.Enabled = false;
            Application.DoEvents();

            image = new Mat(image_path);
            result_image = image.Clone();

            dt1 = DateTime.Now;
            List head_boxes = face_det.Detect(image);

            foreach (var item in head_boxes)
            {
                Rect rect = Rect.FromLTRB((int)item.xmin, (int)item.ymin, (int)item.xmax, (int)item.ymax);
                Cv2.Rectangle(result_image, rect, Scalar.Red);
            }

            List resized_heatmaps = gazelle.Predict(image, head_boxes);
            dt2 = DateTime.Now;

            DrawGaze(result_image, head_boxes, resized_heatmaps);

            pictureBox2.Image = new Bitmap(result_image.ToMemoryStream());
            textBox1.Text = "推理耗时:" + (dt2 - dt1).TotalMilliseconds + "ms";

            button2.Enabled = true;
        }

        void DrawGaze(Mat frame, List head_boxes, List heatmaps, float thr = 0.0f)
        {
            int num_box = head_boxes.Count;
            for (int i = 0; i < num_box; i++)
            {
                double max_score;
                OpenCvSharp.Point classIdPoint;
                double minVal;
                OpenCvSharp.Point minLoc;
                Cv2.MinMaxLoc(heatmaps[i], out minVal, out max_score, out minLoc, out classIdPoint);
                int cx = classIdPoint.X;
                int cy = classIdPoint.Y;
                if (max_score >= thr)
                {
                    int head_cx = (int)((head_boxes[i].xmin + head_boxes[i].xmax) * 0.5);
                    int head_cy = (int)((head_boxes[i].ymin + head_boxes[i].ymax) * 0.5);
                   
                    Cv2.ArrowedLine(frame, new OpenCvSharp.Point(head_cx, head_cy), new OpenCvSharp.Point(cx, cy), new Scalar(0, 255, 0), 2, LineTypes.AntiAlias);
                }
            }
        }

        private void Form1_Load(object sender, EventArgs e)
        {
            face_det = new FaceDet("model\\det_face.onnx");
            gazelle = new GazeLLE("model\\gazelle_dinov2_vitl14_inout_1x3x448x448_1xNx4.onnx");

            image_path = "test_img\\1.jpg";
            pictureBox1.Image = new Bitmap(image_path);
        }

        private void button3_Click(object sender, EventArgs e)
        {
            if (pictureBox2.Image == null)
            {
                return;
            }
            Bitmap output = new Bitmap(pictureBox2.Image);
            SaveFileDialog sdf = new SaveFileDialog();
            sdf.Title = "保存";
            sdf.Filter = "Images (*.jpg)|*.jpg|Images (*.png)|*.png|Images (*.bmp)|*.bmp|Images (*.emf)|*.emf|Images (*.exif)|*.exif|Images (*.gif)|*.gif|Images (*.ico)|*.ico|Images (*.tiff)|*.tiff|Images (*.wmf)|*.wmf";
            if (sdf.ShowDialog() == DialogResult.OK)
            {
                switch (sdf.FilterIndex)
                {
                    case 1:
                        {
                            output.Save(sdf.FileName, ImageFormat.Jpeg);
                            break;
                        }
                    case 2:
                        {
                            output.Save(sdf.FileName, ImageFormat.Png);
                            break;
                        }
                    case 3:
                        {
                            output.Save(sdf.FileName, ImageFormat.Bmp);
                            break;
                        }
                    case 4:
                        {
                            output.Save(sdf.FileName, ImageFormat.Emf);
                            break;
                        }
                    case 5:
                        {
                            output.Save(sdf.FileName, ImageFormat.Exif);
                            break;
                        }
                    case 6:
                        {
                            output.Save(sdf.FileName, ImageFormat.Gif);
                            break;
                        }
                    case 7:
                        {
                            output.Save(sdf.FileName, ImageFormat.Icon);
                            break;
                        }

                    case 8:
                        {
                            output.Save(sdf.FileName, ImageFormat.Tiff);
                            break;
                        }
                    case 9:
                        {
                            output.Save(sdf.FileName, ImageFormat.Wmf);
                            break;
                        }
                }
                MessageBox.Show("保存成功，位置：" + sdf.FileName);
            }
        }
    }
}

  1. using OpenCvSharp;
  2. using System;
  3. using System.Collections.Generic;
  4. using System.Drawing;
  5. using System.Drawing.Imaging;
  6. using System.Windows.Forms;
  7.  
  8. namespace Onnx_Demo
  9. {
  10.     public partial class Form1 : Form
  11.     {
  12.         public Form1()
  13.         {
  14.             InitializeComponent();
  15.         }
  16.  
  17.         string fileFilter = "*.*|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
  18.         string image_path = "";
  19.         DateTime dt1 = DateTime.Now;
  20.         DateTime dt2 = DateTime.Now;
  21.  
  22.         Mat image;
  23.         Mat result_image;
  24.  
  25.         FaceDet face_det;
  26.         GazeLLE gazelle;
  27.  
  28.         private void button1_Click(object sender, EventArgs e)
  29.         {
  30.             OpenFileDialog ofd = new OpenFileDialog();
  31.             ofd.Filter = fileFilter;
  32.             if (ofd.ShowDialog() != DialogResult.OK) return;
  33.             pictureBox1.Image = null;
  34.             image_path = ofd.FileName;
  35.             pictureBox1.Image = new Bitmap(image_path);
  36.             textBox1.Text = "";
  37.             image = new Mat(image_path);
  38.             pictureBox2.Image = null;
  39.         }
  40.  
  41.         private void button2_Click(object sender, EventArgs e)
  42.         {
  43.             if (image_path == "")
  44.             {
  45.                 return;
  46.             }
  47.  
  48.             button2.Enabled = false;
  49.             Application.DoEvents();
  50.  
  51.             image = new Mat(image_path);
  52.             result_image = image.Clone();
  53.  
  54.             dt1 = DateTime.Now;
  55.             List<Bbox> head_boxes = face_det.Detect(image);
  56.  
  57.             foreach (var item in head_boxes)
  58.             {
  59.                 Rect rect = Rect.FromLTRB((int)item.xmin, (int)item.ymin, (int)item.xmax, (int)item.ymax);
  60.                 Cv2.Rectangle(result_image, rect, Scalar.Red);
  61.             }
  62.  
  63.             List<Mat> resized_heatmaps = gazelle.Predict(image, head_boxes);
  64.             dt2 = DateTime.Now;
  65.  
  66.             DrawGaze(result_image, head_boxes, resized_heatmaps);
  67.  
  68.             pictureBox2.Image = new Bitmap(result_image.ToMemoryStream());
  69.             textBox1.Text = "推理耗时:" + (dt2 - dt1).TotalMilliseconds + "ms";
  70.  
  71.             button2.Enabled = true;
  72.         }
  73.  
  74.         void DrawGaze(Mat frame, List<Bbox> head_boxes, List<Mat> heatmaps, float thr = 0.0f)
  75.         {
  76.             int num_box = head_boxes.Count;
  77.             for (int i = 0; i < num_box; i++)
  78.             {
  79.                 double max_score;
  80.                 OpenCvSharp.Point classIdPoint;
  81.                 double minVal;
  82.                 OpenCvSharp.Point minLoc;
  83.                 Cv2.MinMaxLoc(heatmaps[i], out minVal, out max_score, out minLoc, out classIdPoint);
  84.                 int cx = classIdPoint.X;
  85.                 int cy = classIdPoint.Y;
  86.                 if (max_score >= thr)
  87.                 {
  88.                     int head_cx = (int)((head_boxes[i].xmin + head_boxes[i].xmax) * 0.5);
  89.                     int head_cy = (int)((head_boxes[i].ymin + head_boxes[i].ymax) * 0.5);
  90.                    
  91.                     Cv2.ArrowedLine(frame, new OpenCvSharp.Point(head_cx, head_cy), new OpenCvSharp.Point(cx, cy), new Scalar(0, 255, 0), 2, LineTypes.AntiAlias);
  92.                 }
  93.             }
  94.         }
  95.  
  96.         private void Form1_Load(object sender, EventArgs e)
  97.         {
  98.             face_det = new FaceDet("model\\det_face.onnx");
  99.             gazelle = new GazeLLE("model\\gazelle_dinov2_vitl14_inout_1x3x448x448_1xNx4.onnx");
  100.  
  101.             image_path = "test_img\\1.jpg";
  102.             pictureBox1.Image = new Bitmap(image_path);
  103.         }
  104.  
  105.         private void button3_Click(object sender, EventArgs e)
  106.         {
  107.             if (pictureBox2.Image == null)
  108.             {
  109.                 return;
  110.             }
  111.             Bitmap output = new Bitmap(pictureBox2.Image);
  112.             SaveFileDialog sdf = new SaveFileDialog();
  113.             sdf.Title = "保存";
  114.             sdf.Filter = "Images (*.jpg)|*.jpg|Images (*.png)|*.png|Images (*.bmp)|*.bmp|Images (*.emf)|*.emf|Images (*.exif)|*.exif|Images (*.gif)|*.gif|Images (*.ico)|*.ico|Images (*.tiff)|*.tiff|Images (*.wmf)|*.wmf";
  115.             if (sdf.ShowDialog() == DialogResult.OK)
  116.             {
  117.                 switch (sdf.FilterIndex)
  118.                 {
  119.                     case 1:
  120.                         {
  121.                             output.Save(sdf.FileName, ImageFormat.Jpeg);
  122.                             break;
  123.                         }
  124.                     case 2:
  125.                         {
  126.                             output.Save(sdf.FileName, ImageFormat.Png);
  127.                             break;
  128.                         }
  129.                     case 3:
  130.                         {
  131.                             output.Save(sdf.FileName, ImageFormat.Bmp);
  132.                             break;
  133.                         }
  134.                     case 4:
  135.                         {
  136.                             output.Save(sdf.FileName, ImageFormat.Emf);
  137.                             break;
  138.                         }
  139.                     case 5:
  140.                         {
  141.                             output.Save(sdf.FileName, ImageFormat.Exif);
  142.                             break;
  143.                         }
  144.                     case 6:
  145.                         {
  146.                             output.Save(sdf.FileName, ImageFormat.Gif);
  147.                             break;
  148.                         }
  149.                     case 7:
  150.                         {
  151.                             output.Save(sdf.FileName, ImageFormat.Icon);
  152.                             break;
  153.                         }
  154.  
  155.                     case 8:
  156.                         {
  157.                             output.Save(sdf.FileName, ImageFormat.Tiff);
  158.                             break;
  159.                         }
  160.                     case 9:
  161.                         {
  162.                             output.Save(sdf.FileName, ImageFormat.Wmf);
  163.                             break;
  164.                         }
  165.                 }
  166.                 MessageBox.Show("保存成功，位置：" + sdf.FileName);
  167.             }
  168.         }
  169.     }
  170. }

下载

源码下载

参考

https://github.com/hpc203/Gaze-LLE-onnxrun

天天代码码天天
微信公众号
.NET 人工智能实践
注：本文转载自blog.csdn.net的天天代码码天天的文章"https://lw112190.blog.csdn.net/article/details/144559165"。版权归原作者所有，此博客不拥有其著作权，亦不承担相应法律责任。如有侵权，请联系我们删除。
复制链接
相关推荐
发表评论

评论记录：

未查询到任何数据！