OnnxRuntime部署LivePortrait实现快速、高质量的人像驱动视频生成

目录

说明

效果

代码

说明

官网地址：https://github.com/KwaiVGI/LivePortrait

代码地址：https://github.com/hpc203/liveportrait-onnxrun

模型下载：onnx文件在百度云盘，链接: https://pan.baidu.com/s/13wjBFRHIIyCyBsgnBOKqsw 提取码: si95

效果

LivePortrait实现快速、高质量的人像驱动视频生成

代码

main.cpp

#define _CRT_SECURE_NO_WARNINGS
#include
#include
#include
#include "liveportrait.h"

using namespace cv;
using namespace std;


int main()
{
    LivePortraitPipeline mynet;
    
    string imgpath = "/home/wangbo/liveportrait-onnxrun/0.jpg";
    string videopath = "/home/wangbo/liveportrait-onnxrun/d0.mp4";
    
    mynet.execute(imgpath, videopath);

    return 0;
}

liveportrait.cpp

#include "liveportrait.h"
#include
#include

using namespace cv;
using namespace std;
using namespace Ort;

LivePortraitPipeline::LivePortraitPipeline()
{
    /// OrtStatus* status = OrtSessionOptionsAppendExecutionProvider_CUDA(sessionOptions, 0);   ///如果使用cuda加速，需要取消注释
    sessionOptions.SetGraphOptimizationLevel(ORT_ENABLE_BASIC);

    string model_path = "/home/wangbo/liveportrait-onnxrun/weights/appearance_feature_extractor.onnx";
    appearance_feature_extractor_ort_session = new Session(env, model_path.c_str(), sessionOptions);

    model_path = "/home/wangbo/liveportrait-onnxrun/weights/motion_extractor.onnx";
    motion_extractor_ort_session = new Session(env, model_path.c_str(), sessionOptions);

    model_path = "/home/wangbo/liveportrait-onnxrun/weights/warping_spade.onnx";
    warping_spade_ort_session = new Session(env, model_path.c_str(), sessionOptions);

    model_path = "/home/wangbo/liveportrait-onnxrun/weights/stitching.onnx";
    stitching_module_ort_session = new Session(env, model_path.c_str(), sessionOptions);

    model_path = "/home/wangbo/liveportrait-onnxrun/weights/landmark.onnx";
    /// std::wstring widestr = std::wstring(model_path.begin(), model_path.end());  windows写法
    /// landmark_runner_ort_session = new Session(env, widestr.c_str(), sessionOptions); windows写法
    landmark_runner_ort_session = new Session(env, model_path.c_str(), sessionOptions); linux写法
    /// 输出和输出节点名称在头文件里写死,在这里就不调用函数获取了

    this->face_analysis = std::make_shared("/home/wangbo/liveportrait-onnxrun/weights/retinaface_det_static.onnx", "/home/wangbo/liveportrait-onnxrun/weights/face_2dpose_106_static.onnx");

    this->mask_crop = imread("/home/wangbo/liveportrait-onnxrun/mask_template.png");
    cvtColor(this->mask_crop, this->mask_crop, COLOR_BGRA2BGR);
}

Mat LivePortraitPipeline::landmark_runner(const Mat &img, const float *lmk)
{
    std::map crop_dct;
    crop_image(img, lmk, 106, crop_dct, 224, 1.5, -0.1);

    std::vector input_img_shape = {1, 3, crop_dct["img_crop"].rows, crop_dct["img_crop"].cols};
    vector bgrChannels(3);
    split(crop_dct["img_crop"], bgrChannels);
    for (int c = 0; c < 3; c++)
    {
        bgrChannels[c].convertTo(bgrChannels[c], CV_32FC1, 1 / 255.0);
    }
    const int image_area = input_img_shape[2] * input_img_shape[3];
    this->landmark_runner_input_tensor.clear();
    this->landmark_runner_input_tensor.resize(input_img_shape[0] * input_img_shape[1] * image_area);
    size_t single_chn_size = image_area * sizeof(float);
    memcpy(this->landmark_runner_input_tensor.data(), (float *)bgrChannels[0].data, single_chn_size);
    memcpy(this->landmark_runner_input_tensor.data() + image_area, (float *)bgrChannels[1].data, single_chn_size);
    memcpy(this->landmark_runner_input_tensor.data() + image_area * 2, (float *)bgrChannels[2].data, single_chn_size);

    Value input_tensor = Value::CreateTensor(memory_info_handler, this->landmark_runner_input_tensor.data(), this->landmark_runner_input_tensor.size(), input_img_shape.data(), input_img_shape.size());
    vector ort_outputs = this->landmark_runner_ort_session->Run(runOptions, this->landmark_runner_input_names.data(), &input_tensor, 1, this->landmark_runner_output_names.data(), this->landmark_runner_output_names.size());
    float *out_pts = ort_outputs[2].GetTensorMutableData();
    const int num_pts = ort_outputs[2].GetTensorTypeAndShapeInfo().GetShape()[1] / 2;
    Mat lmk_mat(num_pts, 2, CV_32FC1);
    for (int i = 0; i < num_pts; i++)
    { btachsize=1，不考虑batchsize维度
        lmk_mat.at(i, 0) = out_pts[i * 2] * 224 * crop_dct["M_c2o"].at(0, 0) + out_pts[i * 2 + 1] * 224 * crop_dct["M_c2o"].at(0, 1) + crop_dct["M_c2o"].at(0, 2);
        lmk_mat.at(i, 1) = out_pts[i * 2] * 224 * crop_dct["M_c2o"].at(1, 0) + out_pts[i * 2 + 1] * 224 * crop_dct["M_c2o"].at(1, 1) + crop_dct["M_c2o"].at(1, 2);
    }
    return lmk_mat;
}

void LivePortraitPipeline::crop_src_image(const Mat &srcimg, std::map &crop_info)
{
    vector boxes = this->face_analysis->detect(srcimg);
    if (boxes.size() == 0)
    {
        cout << "No face detected in the source image." << endl;
        return;
    }
    else if (boxes.size() > 1)
    {
        cout << "More than one face detected in the image, only pick one face." << endl;
        return;
    }

    Bbox src_face = boxes[0];
    float *lmk = src_face.landmark_2d_106;

    crop_image(srcimg, lmk, 106, crop_info, 512, 2.3, -0.125);

    Mat lmk_crop = this->landmark_runner(srcimg, lmk);
    
    crop_info["lmk_crop"] = lmk_crop;
    Mat img_crop_256x256;
    resize(crop_info["img_crop"], img_crop_256x256, Size(256, 256), INTER_AREA);
    crop_info["img_crop_256x256"] = img_crop_256x256;
    crop_info["lmk_crop_256x256"] = crop_info["lmk_crop"] * 256 / 512;
}

void LivePortraitPipeline::get_kp_info(vector x, vector shape, std::map &kp_info)
{
    Value input_tensor = Value::CreateTensor(memory_info_handler, x.data(), x.size(), shape.data(), shape.size());
    vector ort_outputs = this->motion_extractor_ort_session->Run(runOptions, this->motion_extractor_input_names.data(), &input_tensor, 1, this->motion_extractor_output_names.data(), this->motion_extractor_output_names.size());
    pitch, yaw, roll, t, exp, scale, kp
    float *pitch = ort_outputs[0].GetTensorMutableData();
    float *yaw = ort_outputs[1].GetTensorMutableData();
    float *roll = ort_outputs[2].GetTensorMutableData();
    float *t = ort_outputs[3].GetTensorMutableData();
    float *exp = ort_outputs[4].GetTensorMutableData();
    float *scale = ort_outputs[5].GetTensorMutableData();
    float *kp = ort_outputs[6].GetTensorMutableData();

    vector pred;
    softmax(pitch, this->motion_extractor_output_shape[0], pred);
    const int bs = this->motion_extractor_output_shape[0][0];  batchsize=1,不考虑多图片输入
    const int len = 66; 66
    float sum = 0;
    for (int i = 0; i < len; i++)
    {
        sum += (i * pred[i]);
    }
    float degree = sum * 3 - 97.5;
    kp_info["pitch"] = (Mat_(1, 1) << degree);

    softmax(yaw, this->motion_extractor_output_shape[1], pred);
    sum = 0;
    for (int i = 0; i < len; i++)
    {
        sum += (i * pred[i]);
    }
    degree = sum * 3 - 97.5;
    kp_info["yaw"] = (Mat_(1, 1) << degree);

    softmax(roll, this->motion_extractor_output_shape[2], pred);
    sum = 0;
    for (int i = 0; i < len; i++)
    {
        sum += (i * pred[i]);
    }
    degree = sum * 3 - 97.5;
    kp_info["roll"] = (Mat_(1, 1) << degree);

    kp_info["t"] = Mat(1, 3, CV_32FC1, t);
    vector sizes = {1, 21, 3};   由于在c++的opencv里不支持3维Mat的矩阵乘法,此处不考虑batchsize维度
    vector sizes = {21, 3};
    kp_info["exp"] = Mat(sizes, CV_32FC1, exp);
    kp_info["scale"] = Mat(1, 1, CV_32FC1, scale);
    kp_info["kp"] = Mat(sizes, CV_32FC1, kp);
}

void LivePortraitPipeline::extract_feature_3d(vector x, vector shape, vector &f_s)
{
    Value input_tensor = Value::CreateTensor(memory_info_handler, x.data(), x.size(), shape.data(), shape.size());
    vector ort_outputs = this->appearance_feature_extractor_ort_session->Run(runOptions, this->appearance_feature_extractor_input_names.data(), &input_tensor, 1, this->appearance_feature_extractor_output_names.data(), this->appearance_feature_extractor_output_names.size());
    int numel = ort_outputs[0].GetTensorTypeAndShapeInfo().GetElementCount();
    float *out = ort_outputs[0].GetTensorMutableData();

    f_s.clear();
    f_s.resize(numel);
    memcpy(f_s.data(), out, numel * sizeof(float));
}

void LivePortraitPipeline::stitching(const Mat &kp_source, Mat &kp_driving_new)
{
    不考虑batchsize维度
    const int num_kp = kp_source.size[0];
    const int numel = kp_source.total();

    const int len = std::accumulate(this->stitching_module_input_shape.begin(), this->stitching_module_input_shape.end(), 1, std::multiplies());
    vector feat(len);
    memcpy(feat.data(), (float *)kp_source.data, numel * sizeof(float));
    memcpy(feat.data() + numel, (float *)kp_driving_new.data, (len - numel) * sizeof(float));

    Value input_tensor = Value::CreateTensor(memory_info_handler, feat.data(), feat.size(), this->stitching_module_input_shape.data(), this->stitching_module_input_shape.size());
    vector ort_outputs = this->stitching_module_ort_session->Run(runOptions, this->stitching_module_input_names.data(), &input_tensor, 1, this->stitching_module_output_names.data(), this->stitching_module_output_names.size());
    float *delta = ort_outputs[0].GetTensorMutableData();
    const float delta_tx_ty[2] = {delta[num_kp * 3], delta[num_kp * 3 + 1]};
    for (int i = 0; i < num_kp; i++)
    {
        kp_driving_new.at(i, 0) += delta[i * 3];
        kp_driving_new.at(i, 1) += delta[i * 3 + 1];
        kp_driving_new.at(i, 2) += delta[i * 3 + 2];

        kp_driving_new.at(i, 0) += delta_tx_ty[0];
        kp_driving_new.at(i, 1) += delta_tx_ty[1];
    }
}

Mat LivePortraitPipeline::warping_spade(vector feature_3d, const Mat &kp_source, const Mat &kp_driving)
{
    vector inputTensors;
    inputTensors.emplace_back(Ort::Value::CreateTensor(memory_info_handler, feature_3d.data(), feature_3d.size(), this->warping_spade_input_shape[0].data(), this->warping_spade_input_shape[0].size()));
    inputTensors.emplace_back(Ort::Value::CreateTensor(memory_info_handler, (float *)kp_driving.data, kp_driving.total(), this->warping_spade_input_shape[1].data(), this->warping_spade_input_shape[1].size()));
    inputTensors.emplace_back(Ort::Value::CreateTensor(memory_info_handler, (float *)kp_source.data, kp_source.total(), this->warping_spade_input_shape[2].data(), this->warping_spade_input_shape[2].size()));
    
    vector ort_outputs = this->warping_spade_ort_session->Run(runOptions, this->warping_spade_input_names.data(), inputTensors.data(), inputTensors.size(), this->warping_spade_output_names.data(), this->warping_spade_output_names.size());
    float *out = ort_outputs[0].GetTensorMutableData();
    Mat out_mat = Mat(this->warping_spade_output_shape, CV_32FC1, out);
    return out_mat;
}

Mat LivePortraitPipeline::predict(const int frame_id, std::map x_s_info, const Mat &R_s, vector f_s, const Mat &x_s, const Mat &frame)
{
    Mat lmk;
    if (frame_id > 0)
    {
        lmk = this->landmark_runner(frame, (float*)this->pred_info_lmk.data);
    }
    else
    {
        vector boxes = this->face_analysis->detect(frame);
        if (boxes.size() == 0)
        {
            cout << "No face detected in the frame." << endl;
            exit(-1);
        }
        else if (boxes.size() > 1)
        {
            cout << "More than one face detected in the driving frame, only pick one face." << endl;
            exit(-1);
        }
        Bbox src_face = boxes[0];
        lmk = this->landmark_runner(frame, src_face.landmark_2d_106);
    }
    lmk.copyTo(this->pred_info_lmk);

    float c_d_eyes[2] = {calculate_distance_ratio(lmk, 6, 18, 0, 12), calculate_distance_ratio(lmk, 30, 42, 24, 36)};
    float c_d_lip = calculate_distance_ratio(lmk, 90, 102, 48, 66);

    Mat img;
    resize(frame, img, Size(256, 256));
    vector I_d;
    vector I_d_shape;
    preprocess(img, I_d, I_d_shape);

    std::map x_d_info;
    this->get_kp_info(I_d, I_d_shape, x_d_info);
    Mat R_d = get_rotation_matrix(x_d_info["pitch"], x_d_info["yaw"], x_d_info["roll"]);
    x_d_info["R_d"] = R_d;
    x_d_info.erase("pitch");
    x_d_info.erase("yaw");
    x_d_info.erase("roll");
    x_d_info.erase("kp");

    if (frame_id == 0)
    {
        this->pred_info_x_d_0_info["scale"] = x_d_info["scale"].clone();   ///也可以定义结构体的方式打包参数
        this->pred_info_x_d_0_info["R_d"] = R_d.clone();
        this->pred_info_x_d_0_info["exp"] = x_d_info["exp"].clone();
        this->pred_info_x_d_0_info["t"] = x_d_info["t"].clone();
    }

    Mat R_new = (R_d * this->pred_info_x_d_0_info["R_d"].t()) * R_s;
    Mat delta_new = x_s_info["exp"] + (x_d_info["exp"] - this->pred_info_x_d_0_info["exp"]);
    Mat scale_new = x_s_info["scale"].mul(x_d_info["scale"] / this->pred_info_x_d_0_info["scale"]); /// scale是1x1矩阵,也就是单个数值
    Mat t_new = x_s_info["t"] + (x_d_info["t"] - this->pred_info_x_d_0_info["t"]);
    
    t_new.at(0, 2) = 0;
    Mat temp = repeat(t_new, 21, 1);
    Mat x_d_new = scale_new.at(0, 0) * (x_s_info["kp"] * R_new + delta_new) + temp;
    
    this->stitching(x_s, x_d_new);
    
    Mat out = this->warping_spade(f_s, x_s, x_d_new);   形状是[1,3,512,512]
    const int image_erea = out.size[2] * out.size[3];
    float *pdata = (float *)out.data;
    Mat rmat = Mat(out.size[2], out.size[3], CV_32FC1, pdata);
    Mat gmat = Mat(out.size[2], out.size[3], CV_32FC1, pdata + image_erea);
    Mat bmat = Mat(out.size[2], out.size[3], CV_32FC1, pdata + image_erea * 2);
    rmat.setTo(0, rmat < 0);
    rmat.setTo(1, rmat > 1);
    gmat.setTo(0, gmat < 0);
    gmat.setTo(1, gmat > 1);
    bmat.setTo(0, bmat < 0);
    bmat.setTo(1, bmat > 1);
    vector channel_mats(3);
    channel_mats[0] = rmat;
    channel_mats[1] = gmat;
    channel_mats[2] = bmat;
    Mat I_p;
    merge(channel_mats, I_p);
    I_p *= 255;
    I_p.convertTo(I_p, CV_8UC3);
    return I_p;
}

int LivePortraitPipeline::execute(string imgpath, string videopath)
{
    Mat srcimg = imread(imgpath);
    if (srcimg.empty())
    {
        cout << "opencv读取图片为空, 请检查输入图片的路径" << endl;
        return -1;
    }

    Mat img;
    cvtColor(srcimg, img, COLOR_BGRA2BGR);
    cvtColor(img, img, COLOR_BGR2RGB);
    Mat src_img = src_preprocess(img);
    std::map crop_info;
    crop_src_image(src_img, crop_info);

    Mat img_crop_256x256 = crop_info["img_crop_256x256"];
    vector I_s;
    vector I_s_shape;
    preprocess(img_crop_256x256, I_s, I_s_shape);

    std::map x_s_info;
    this->get_kp_info(I_s, I_s_shape, x_s_info);
    Mat R_s = get_rotation_matrix(x_s_info["pitch"], x_s_info["yaw"], x_s_info["roll"]);  返回结果已验证通过

    vector f_s;
    this->extract_feature_3d(I_s, I_s_shape, f_s);
    Mat x_s = transform_keypoint(x_s_info);

    cv::VideoCapture capture(videopath);
    if (!capture.isOpened())
    {
        cout << "VideoCapture,open video file failed, " << videopath << endl;
        return -1;
    }
    const int fps = capture.get(cv::CAP_PROP_FPS);
    const int video_length = capture.get(cv::CAP_PROP_FRAME_COUNT);
    cout<<"video total have "<     int f_h = src_img.rows;
    int f_w = src_img.cols;
    if (this->flg_composite)
    {
        f_h = 512;
        f_w = 512 * 3;
    }

    prepare for pasteback
    Mat mask_ori = prepare_paste_back(this->mask_crop, crop_info["M_c2o"], Size(src_img.cols, src_img.rows));

    VideoWriter video_writer;
    video_writer.open("output.mp4", cv::VideoWriter::fourcc('m', 'p', '4', 'v'), fps, Size(f_w, f_h));
    Mat frame;
    int frame_id = 0;
    while (capture.read(frame))
    {
        if (frame.empty())
        {
            break;
        }

        Mat img_rgb;
        cvtColor(frame, img_rgb, COLOR_BGR2RGB);
        auto a = std::chrono::high_resolution_clock::now();
        Mat I_p = this->predict(frame_id, x_s_info, R_s, f_s, x_s, img_rgb);
        auto b = std::chrono::high_resolution_clock::now();
        std::chrono::duration c = b - a;
        cout<<"frame_id="<         
        frame_id += 1;
        Mat driving_img;
        if (this->flg_composite)
        {
            concat_frame(img_rgb, img_crop_256x256, I_p, driving_img);
        }
        else
        {
            paste_back(I_p, crop_info["M_c2o"], src_img, mask_ori, driving_img);
        }
        cvtColor(driving_img, driving_img, COLOR_RGB2BGR);
        video_writer.write(driving_img);
    }
    video_writer.release();
    capture.release();
    ///destroyAllWindows();
    return 0;
}

  1. #include "liveportrait.h"
  2. #include <opencv2/highgui.hpp>
  3. #include <numeric>
  4.  
  5. using namespace cv;
  6. using namespace std;
  7. using namespace Ort;
  8.  
  9. LivePortraitPipeline::LivePortraitPipeline()
  10. {
  11.     /// OrtStatus* status = OrtSessionOptionsAppendExecutionProvider_CUDA(sessionOptions, 0);   ///如果使用cuda加速，需要取消注释
  12.     sessionOptions.SetGraphOptimizationLevel(ORT_ENABLE_BASIC);
  13.  
  14.     string model_path = "/home/wangbo/liveportrait-onnxrun/weights/appearance_feature_extractor.onnx";
  15.     appearance_feature_extractor_ort_session = new Session(env, model_path.c_str(), sessionOptions);
  16.  
  17.     model_path = "/home/wangbo/liveportrait-onnxrun/weights/motion_extractor.onnx";
  18.     motion_extractor_ort_session = new Session(env, model_path.c_str(), sessionOptions);
  19.  
  20.     model_path = "/home/wangbo/liveportrait-onnxrun/weights/warping_spade.onnx";
  21.     warping_spade_ort_session = new Session(env, model_path.c_str(), sessionOptions);
  22.  
  23.     model_path = "/home/wangbo/liveportrait-onnxrun/weights/stitching.onnx";
  24.     stitching_module_ort_session = new Session(env, model_path.c_str(), sessionOptions);
  25.  
  26.     model_path = "/home/wangbo/liveportrait-onnxrun/weights/landmark.onnx";
  27.     /// std::wstring widestr = std::wstring(model_path.begin(), model_path.end());  windows写法
  28.     /// landmark_runner_ort_session = new Session(env, widestr.c_str(), sessionOptions); windows写法
  29.     landmark_runner_ort_session = new Session(env, model_path.c_str(), sessionOptions); linux写法
  30.     /// 输出和输出节点名称在头文件里写死,在这里就不调用函数获取了
  31.  
  32.     this->face_analysis = std::make_shared<FaceAnalysis>("/home/wangbo/liveportrait-onnxrun/weights/retinaface_det_static.onnx", "/home/wangbo/liveportrait-onnxrun/weights/face_2dpose_106_static.onnx");
  33.  
  34.     this->mask_crop = imread("/home/wangbo/liveportrait-onnxrun/mask_template.png");
  35.     cvtColor(this->mask_crop, this->mask_crop, COLOR_BGRA2BGR);
  36. }
  37.  
  38. Mat LivePortraitPipeline::landmark_runner(const Mat &img, const float *lmk)
  39. {
  40.     std::map<string, Mat> crop_dct;
  41.     crop_image(img, lmk, 106, crop_dct, 224, 1.5, -0.1);
  42.  
  43.     std::vector<int64_t> input_img_shape = {1, 3, crop_dct["img_crop"].rows, crop_dct["img_crop"].cols};
  44.     vector<cv::Mat> bgrChannels(3);
  45.     split(crop_dct["img_crop"], bgrChannels);
  46.     for (int c = 0; c < 3; c++)
  47.     {
  48.         bgrChannels[c].convertTo(bgrChannels[c], CV_32FC1, 1 / 255.0);
  49.     }
  50.     const int image_area = input_img_shape[2] * input_img_shape[3];
  51.     this->landmark_runner_input_tensor.clear();
  52.     this->landmark_runner_input_tensor.resize(input_img_shape[0] * input_img_shape[1] * image_area);
  53.     size_t single_chn_size = image_area * sizeof(float);
  54.     memcpy(this->landmark_runner_input_tensor.data(), (float *)bgrChannels[0].data, single_chn_size);
  55.     memcpy(this->landmark_runner_input_tensor.data() + image_area, (float *)bgrChannels[1].data, single_chn_size);
  56.     memcpy(this->landmark_runner_input_tensor.data() + image_area * 2, (float *)bgrChannels[2].data, single_chn_size);
  57.  
  58.     Value input_tensor = Value::CreateTensor<float>(memory_info_handler, this->landmark_runner_input_tensor.data(), this->landmark_runner_input_tensor.size(), input_img_shape.data(), input_img_shape.size());
  59.     vector<Value> ort_outputs = this->landmark_runner_ort_session->Run(runOptions, this->landmark_runner_input_names.data(), &input_tensor, 1, this->landmark_runner_output_names.data(), this->landmark_runner_output_names.size());
  60.     float *out_pts = ort_outputs[2].GetTensorMutableData<float>();
  61.     const int num_pts = ort_outputs[2].GetTensorTypeAndShapeInfo().GetShape()[1] / 2;
  62.     Mat lmk_mat(num_pts, 2, CV_32FC1);
  63.     for (int i = 0; i < num_pts; i++)
  64.     { btachsize=1，不考虑batchsize维度
  65.         lmk_mat.at<float>(i, 0) = out_pts[i * 2] * 224 * crop_dct["M_c2o"].at<float>(0, 0) + out_pts[i * 2 + 1] * 224 * crop_dct["M_c2o"].at<float>(0, 1) + crop_dct["M_c2o"].at<float>(0, 2);
  66.         lmk_mat.at<float>(i, 1) = out_pts[i * 2] * 224 * crop_dct["M_c2o"].at<float>(1, 0) + out_pts[i * 2 + 1] * 224 * crop_dct["M_c2o"].at<float>(1, 1) + crop_dct["M_c2o"].at<float>(1, 2);
  67.     }
  68.     return lmk_mat;
  69. }
  70.  
  71. void LivePortraitPipeline::crop_src_image(const Mat &srcimg, std::map<string, Mat> &crop_info)
  72. {
  73.     vector<Bbox> boxes = this->face_analysis->detect(srcimg);
  74.     if (boxes.size() == 0)
  75.     {
  76.         cout << "No face detected in the source image." << endl;
  77.         return;
  78.     }
  79.     else if (boxes.size() > 1)
  80.     {
  81.         cout << "More than one face detected in the image, only pick one face." << endl;
  82.         return;
  83.     }
  84.  
  85.     Bbox src_face = boxes[0];
  86.     float *lmk = src_face.landmark_2d_106;
  87.  
  88.     crop_image(srcimg, lmk, 106, crop_info, 512, 2.3, -0.125);
  89.  
  90.     Mat lmk_crop = this->landmark_runner(srcimg, lmk);
  91.     
  92.     crop_info["lmk_crop"] = lmk_crop;
  93.     Mat img_crop_256x256;
  94.     resize(crop_info["img_crop"], img_crop_256x256, Size(256, 256), INTER_AREA);
  95.     crop_info["img_crop_256x256"] = img_crop_256x256;
  96.     crop_info["lmk_crop_256x256"] = crop_info["lmk_crop"] * 256 / 512;
  97. }
  98.  
  99. void LivePortraitPipeline::get_kp_info(vector<float> x, vector<int64_t> shape, std::map<string, Mat> &kp_info)
  100. {
  101.     Value input_tensor = Value::CreateTensor<float>(memory_info_handler, x.data(), x.size(), shape.data(), shape.size());
  102.     vector<Value> ort_outputs = this->motion_extractor_ort_session->Run(runOptions, this->motion_extractor_input_names.data(), &input_tensor, 1, this->motion_extractor_output_names.data(), this->motion_extractor_output_names.size());
  103.     pitch, yaw, roll, t, exp, scale, kp
  104.     float *pitch = ort_outputs[0].GetTensorMutableData<float>();
  105.     float *yaw = ort_outputs[1].GetTensorMutableData<float>();
  106.     float *roll = ort_outputs[2].GetTensorMutableData<float>();
  107.     float *t = ort_outputs[3].GetTensorMutableData<float>();
  108.     float *exp = ort_outputs[4].GetTensorMutableData<float>();
  109.     float *scale = ort_outputs[5].GetTensorMutableData<float>();
  110.     float *kp = ort_outputs[6].GetTensorMutableData<float>();
  111.  
  112.     vector<float> pred;
  113.     softmax(pitch, this->motion_extractor_output_shape[0], pred);
  114.     const int bs = this->motion_extractor_output_shape[0][0];  batchsize=1,不考虑多图片输入
  115.     const int len = 66; 66
  116.     float sum = 0;
  117.     for (int i = 0; i < len; i++)
  118.     {
  119.         sum += (i * pred[i]);
  120.     }
  121.     float degree = sum * 3 - 97.5;
  122.     kp_info["pitch"] = (Mat_<float>(1, 1) << degree);
  123.  
  124.     softmax(yaw, this->motion_extractor_output_shape[1], pred);
  125.     sum = 0;
  126.     for (int i = 0; i < len; i++)
  127.     {
  128.         sum += (i * pred[i]);
  129.     }
  130.     degree = sum * 3 - 97.5;
  131.     kp_info["yaw"] = (Mat_<float>(1, 1) << degree);
  132.  
  133.     softmax(roll, this->motion_extractor_output_shape[2], pred);
  134.     sum = 0;
  135.     for (int i = 0; i < len; i++)
  136.     {
  137.         sum += (i * pred[i]);
  138.     }
  139.     degree = sum * 3 - 97.5;
  140.     kp_info["roll"] = (Mat_<float>(1, 1) << degree);
  141.  
  142.     kp_info["t"] = Mat(1, 3, CV_32FC1, t);
  143.     vector<int> sizes = {1, 21, 3};   由于在c++的opencv里不支持3维Mat的矩阵乘法,此处不考虑batchsize维度
  144.     vector<int> sizes = {21, 3};
  145.     kp_info["exp"] = Mat(sizes, CV_32FC1, exp);
  146.     kp_info["scale"] = Mat(1, 1, CV_32FC1, scale);
  147.     kp_info["kp"] = Mat(sizes, CV_32FC1, kp);
  148. }
  149.  
  150. void LivePortraitPipeline::extract_feature_3d(vector<float> x, vector<int64_t> shape, vector<float> &f_s)
  151. {
  152.     Value input_tensor = Value::CreateTensor<float>(memory_info_handler, x.data(), x.size(), shape.data(), shape.size());
  153.     vector<Value> ort_outputs = this->appearance_feature_extractor_ort_session->Run(runOptions, this->appearance_feature_extractor_input_names.data(), &input_tensor, 1, this->appearance_feature_extractor_output_names.data(), this->appearance_feature_extractor_output_names.size());
  154.     int numel = ort_outputs[0].GetTensorTypeAndShapeInfo().GetElementCount();
  155.     float *out = ort_outputs[0].GetTensorMutableData<float>();
  156.  
  157.     f_s.clear();
  158.     f_s.resize(numel);
  159.     memcpy(f_s.data(), out, numel * sizeof(float));
  160. }
  161.  
  162. void LivePortraitPipeline::stitching(const Mat &kp_source, Mat &kp_driving_new)
  163. {
  164.     不考虑batchsize维度
  165.     const int num_kp = kp_source.size[0];
  166.     const int numel = kp_source.total();
  167.  
  168.     const int len = std::accumulate(this->stitching_module_input_shape.begin(), this->stitching_module_input_shape.end(), 1, std::multiplies<int64_t>());
  169.     vector<float> feat(len);
  170.     memcpy(feat.data(), (float *)kp_source.data, numel * sizeof(float));
  171.     memcpy(feat.data() + numel, (float *)kp_driving_new.data, (len - numel) * sizeof(float));
  172.  
  173.     Value input_tensor = Value::CreateTensor<float>(memory_info_handler, feat.data(), feat.size(), this->stitching_module_input_shape.data(), this->stitching_module_input_shape.size());
  174.     vector<Value> ort_outputs = this->stitching_module_ort_session->Run(runOptions, this->stitching_module_input_names.data(), &input_tensor, 1, this->stitching_module_output_names.data(), this->stitching_module_output_names.size());
  175.     float *delta = ort_outputs[0].GetTensorMutableData<float>();
  176.     const float delta_tx_ty[2] = {delta[num_kp * 3], delta[num_kp * 3 + 1]};
  177.     for (int i = 0; i < num_kp; i++)
  178.     {
  179.         kp_driving_new.at<float>(i, 0) += delta[i * 3];
  180.         kp_driving_new.at<float>(i, 1) += delta[i * 3 + 1];
  181.         kp_driving_new.at<float>(i, 2) += delta[i * 3 + 2];
  182.  
  183.         kp_driving_new.at<float>(i, 0) += delta_tx_ty[0];
  184.         kp_driving_new.at<float>(i, 1) += delta_tx_ty[1];
  185.     }
  186. }
  187.  
  188. Mat LivePortraitPipeline::warping_spade(vector<float> feature_3d, const Mat &kp_source, const Mat &kp_driving)
  189. {
  190.     vector<Ort::Value> inputTensors;
  191.     inputTensors.emplace_back(Ort::Value::CreateTensor<float>(memory_info_handler, feature_3d.data(), feature_3d.size(), this->warping_spade_input_shape[0].data(), this->warping_spade_input_shape[0].size()));
  192.     inputTensors.emplace_back(Ort::Value::CreateTensor<float>(memory_info_handler, (float *)kp_driving.data, kp_driving.total(), this->warping_spade_input_shape[1].data(), this->warping_spade_input_shape[1].size()));
  193.     inputTensors.emplace_back(Ort::Value::CreateTensor<float>(memory_info_handler, (float *)kp_source.data, kp_source.total(), this->warping_spade_input_shape[2].data(), this->warping_spade_input_shape[2].size()));
  194.     
  195.     vector<Value> ort_outputs = this->warping_spade_ort_session->Run(runOptions, this->warping_spade_input_names.data(), inputTensors.data(), inputTensors.size(), this->warping_spade_output_names.data(), this->warping_spade_output_names.size());
  196.     float *out = ort_outputs[0].GetTensorMutableData<float>();
  197.     Mat out_mat = Mat(this->warping_spade_output_shape, CV_32FC1, out);
  198.     return out_mat;
  199. }
  200.  
  201. Mat LivePortraitPipeline::predict(const int frame_id, std::map<string, Mat> x_s_info, const Mat &R_s, vector<float> f_s, const Mat &x_s, const Mat &frame)
  202. {
  203.     Mat lmk;
  204.     if (frame_id > 0)
  205.     {
  206.         lmk = this->landmark_runner(frame, (float*)this->pred_info_lmk.data);
  207.     }
  208.     else
  209.     {
  210.         vector<Bbox> boxes = this->face_analysis->detect(frame);
  211.         if (boxes.size() == 0)
  212.         {
  213.             cout << "No face detected in the frame." << endl;
  214.             exit(-1);
  215.         }
  216.         else if (boxes.size() > 1)
  217.         {
  218.             cout << "More than one face detected in the driving frame, only pick one face." << endl;
  219.             exit(-1);
  220.         }
  221.         Bbox src_face = boxes[0];
  222.         lmk = this->landmark_runner(frame, src_face.landmark_2d_106);
  223.     }
  224.     lmk.copyTo(this->pred_info_lmk);
  225.  
  226.     float c_d_eyes[2] = {calculate_distance_ratio(lmk, 6, 18, 0, 12), calculate_distance_ratio(lmk, 30, 42, 24, 36)};
  227.     float c_d_lip = calculate_distance_ratio(lmk, 90, 102, 48, 66);
  228.  
  229.     Mat img;
  230.     resize(frame, img, Size(256, 256));
  231.     vector<float> I_d;
  232.     vector<int64_t> I_d_shape;
  233.     preprocess(img, I_d, I_d_shape);
  234.  
  235.     std::map<string, Mat> x_d_info;
  236.     this->get_kp_info(I_d, I_d_shape, x_d_info);
  237.     Mat R_d = get_rotation_matrix(x_d_info["pitch"], x_d_info["yaw"], x_d_info["roll"]);
  238.     x_d_info["R_d"] = R_d;
  239.     x_d_info.erase("pitch");
  240.     x_d_info.erase("yaw");
  241.     x_d_info.erase("roll");
  242.     x_d_info.erase("kp");
  243.  
  244.     if (frame_id == 0)
  245.     {
  246.         this->pred_info_x_d_0_info["scale"] = x_d_info["scale"].clone();   ///也可以定义结构体的方式打包参数
  247.         this->pred_info_x_d_0_info["R_d"] = R_d.clone();
  248.         this->pred_info_x_d_0_info["exp"] = x_d_info["exp"].clone();
  249.         this->pred_info_x_d_0_info["t"] = x_d_info["t"].clone();
  250.     }
  251.  
  252.     Mat R_new = (R_d * this->pred_info_x_d_0_info["R_d"].t()) * R_s;
  253.     Mat delta_new = x_s_info["exp"] + (x_d_info["exp"] - this->pred_info_x_d_0_info["exp"]);
  254.     Mat scale_new = x_s_info["scale"].mul(x_d_info["scale"] / this->pred_info_x_d_0_info["scale"]); /// scale是1x1矩阵,也就是单个数值
  255.     Mat t_new = x_s_info["t"] + (x_d_info["t"] - this->pred_info_x_d_0_info["t"]);
  256.     
  257.     t_new.at<float>(0, 2) = 0;
  258.     Mat temp = repeat(t_new, 21, 1);
  259.     Mat x_d_new = scale_new.at<float>(0, 0) * (x_s_info["kp"] * R_new + delta_new) + temp;
  260.     
  261.     this->stitching(x_s, x_d_new);
  262.     
  263.     Mat out = this->warping_spade(f_s, x_s, x_d_new);   形状是[1,3,512,512]
  264.     const int image_erea = out.size[2] * out.size[3];
  265.     float *pdata = (float *)out.data;
  266.     Mat rmat = Mat(out.size[2], out.size[3], CV_32FC1, pdata);
  267.     Mat gmat = Mat(out.size[2], out.size[3], CV_32FC1, pdata + image_erea);
  268.     Mat bmat = Mat(out.size[2], out.size[3], CV_32FC1, pdata + image_erea * 2);
  269.     rmat.setTo(0, rmat < 0);
  270.     rmat.setTo(1, rmat > 1);
  271.     gmat.setTo(0, gmat < 0);
  272.     gmat.setTo(1, gmat > 1);
  273.     bmat.setTo(0, bmat < 0);
  274.     bmat.setTo(1, bmat > 1);
  275.     vector<Mat> channel_mats(3);
  276.     channel_mats[0] = rmat;
  277.     channel_mats[1] = gmat;
  278.     channel_mats[2] = bmat;
  279.     Mat I_p;
  280.     merge(channel_mats, I_p);
  281.     I_p *= 255;
  282.     I_p.convertTo(I_p, CV_8UC3);
  283.     return I_p;
  284. }
  285.  
  286. int LivePortraitPipeline::execute(string imgpath, string videopath)
  287. {
  288.     Mat srcimg = imread(imgpath);
  289.     if (srcimg.empty())
  290.     {
  291.         cout << "opencv读取图片为空, 请检查输入图片的路径" << endl;
  292.         return -1;
  293.     }
  294.  
  295.     Mat img;
  296.     cvtColor(srcimg, img, COLOR_BGRA2BGR);
  297.     cvtColor(img, img, COLOR_BGR2RGB);
  298.     Mat src_img = src_preprocess(img);
  299.     std::map<string, Mat> crop_info;
  300.     crop_src_image(src_img, crop_info);
  301.  
  302.     Mat img_crop_256x256 = crop_info["img_crop_256x256"];
  303.     vector<float> I_s;
  304.     vector<int64_t> I_s_shape;
  305.     preprocess(img_crop_256x256, I_s, I_s_shape);
  306.  
  307.     std::map<string, Mat> x_s_info;
  308.     this->get_kp_info(I_s, I_s_shape, x_s_info);
  309.     Mat R_s = get_rotation_matrix(x_s_info["pitch"], x_s_info["yaw"], x_s_info["roll"]);  返回结果已验证通过
  310.  
  311.     vector<float> f_s;
  312.     this->extract_feature_3d(I_s, I_s_shape, f_s);
  313.     Mat x_s = transform_keypoint(x_s_info);
  314.  
  315.     cv::VideoCapture capture(videopath);
  316.     if (!capture.isOpened())
  317.     {
  318.         cout << "VideoCapture,open video file failed, " << videopath << endl;
  319.         return -1;
  320.     }
  321.     const int fps = capture.get(cv::CAP_PROP_FPS);
  322.     const int video_length = capture.get(cv::CAP_PROP_FRAME_COUNT);
  323.     cout<<"video total have "<<video_length<<" frames"<<endl;
  324.     int f_h = src_img.rows;
  325.     int f_w = src_img.cols;
  326.     if (this->flg_composite)
  327.     {
  328.         f_h = 512;
  329.         f_w = 512 * 3;
  330.     }
  331.  
  332.     prepare for pasteback
  333.     Mat mask_ori = prepare_paste_back(this->mask_crop, crop_info["M_c2o"], Size(src_img.cols, src_img.rows));
  334.  
  335.     VideoWriter video_writer;
  336.     video_writer.open("output.mp4", cv::VideoWriter::fourcc('m', 'p', '4', 'v'), fps, Size(f_w, f_h));
  337.     Mat frame;
  338.     int frame_id = 0;
  339.     while (capture.read(frame))
  340.     {
  341.         if (frame.empty())
  342.         {
  343.             break;
  344.         }
  345.  
  346.         Mat img_rgb;
  347.         cvtColor(frame, img_rgb, COLOR_BGR2RGB);
  348.         auto a = std::chrono::high_resolution_clock::now();
  349.         Mat I_p = this->predict(frame_id, x_s_info, R_s, f_s, x_s, img_rgb);
  350.         auto b = std::chrono::high_resolution_clock::now();
  351.         std::chrono::duration<double> c = b - a;
  352.         cout<<"frame_id="<<frame_id<<", predict waste time="<<to_string(c.count())<<" s"<<endl;
  353.         
  354.         frame_id += 1;
  355.         Mat driving_img;
  356.         if (this->flg_composite)
  357.         {
  358.             concat_frame(img_rgb, img_crop_256x256, I_p, driving_img);
  359.         }
  360.         else
  361.         {
  362.             paste_back(I_p, crop_info["M_c2o"], src_img, mask_ori, driving_img);
  363.         }
  364.         cvtColor(driving_img, driving_img, COLOR_RGB2BGR);
  365.         video_writer.write(driving_img);
  366.     }
  367.     video_writer.release();
  368.     capture.release();
  369.     ///destroyAllWindows();
  370.     return 0;
  371. }

faceanalysis.cpp

  1. #include "faceanalysis.h"
  2.  
  3.  
  4. using namespace cv;
  5. using namespace std;
  6. using namespace Ort;
  7.  
  8.  
  9. FaceAnalysis::FaceAnalysis(string model_patha, string model_pathb)
  10. {
  11.     /// OrtStatus* status = OrtSessionOptionsAppendExecutionProvider_CUDA(sessionOptions, 0);   ///如果使用cuda加速，需要取消注释
  12.     sessionOptions.SetGraphOptimizationLevel(ORT_ENABLE_BASIC);
  13.     /// std::wstring widestr = std::wstring(model_patha.begin(), model_patha.end());  windows写法
  14.     /// ort_session = new Session(env, widestr.c_str(), sessionOptions); windows写法
  15.     det_face_ort_session = new Session(env, model_patha.c_str(), sessionOptions); linux写法
  16.  
  17.     size_t numInputNodes = det_face_ort_session->GetInputCount();
  18.     size_t numOutputNodes = det_face_ort_session->GetOutputCount();
  19.     AllocatorWithDefaultOptions allocator;
  20.     
  21.     for (int i = 0; i < numOutputNodes; i++)
  22.     {
  23.         Ort::TypeInfo output_type_info = det_face_ort_session->GetOutputTypeInfo(i);
  24.         auto output_tensor_info = output_type_info.GetTensorTypeAndShapeInfo();
  25.         auto output_dims = output_tensor_info.GetShape();
  26.         det_face_output_node_dims.push_back(output_dims);
  27.     }
  28.  
  29.     /// std::wstring widestr = std::wstring(model_pathb.begin(), model_pathb.end());  windows写法
  30.     /// ort_session = new Session(env, widestr.c_str(), sessionOptions); windows写法
  31.     landmark_ort_session = new Session(env, model_pathb.c_str(), sessionOptions); linux写法
  32.     numInputNodes = landmark_ort_session->GetInputCount();
  33.     numOutputNodes = landmark_ort_session->GetOutputCount();
  34.     
  35.     Ort::TypeInfo input_type_info = landmark_ort_session->GetInputTypeInfo(0);
  36. 	auto input_tensor_info = input_type_info.GetTensorTypeAndShapeInfo();
  37. 	auto input_dims = input_tensor_info.GetShape();
  38. 	this->landmark_input_height = input_dims[2];
  39.     this->landmark_input_width = input_dims[3];
  40. 	this->landmark_input_tensor_shape = { 1, 3, this->landmark_input_height, this->landmark_input_width };
  41.     
  42.     Ort::TypeInfo output_type_info = landmark_ort_session->GetOutputTypeInfo(0);
  43.     auto output_tensor_info = output_type_info.GetTensorTypeAndShapeInfo();
  44.     auto output_dims = output_tensor_info.GetShape();
  45.     landmark_output_node_dims.push_back(output_dims);  
  46. }
  47.  
  48. void FaceAnalysis::preprocess(Mat srcimg)
  49. {
  50.     const float im_ratio = float(srcimg.rows) / (float)srcimg.cols;
  51.     int new_width = this->input_size;
  52.     int new_height = int(new_width * im_ratio);
  53.     if(im_ratio>1)
  54.     {
  55.         new_height = this->input_size;
  56.         new_width = int(new_height / im_ratio);
  57.     }
  58.     this->det_scale = float(new_height) / (float)srcimg.rows;
  59.     Mat resized_img;
  60.     resize(srcimg, resized_img, Size(new_width, new_height));
  61.     Mat det_img;
  62.     copyMakeBorder(resized_img, det_img, 0, this->input_size - new_height, 0, this->input_size - new_width, BORDER_CONSTANT, 0);
  63.  
  64.     vector<cv::Mat> bgrChannels(3);
  65.     split(det_img, bgrChannels);
  66.     for (int c = 0; c < 3; c++)
  67.     {
  68.         bgrChannels[c].convertTo(bgrChannels[c], CV_32FC1, 1 / 128.0, -127.5 / 128.0);
  69.     }
  70.  
  71.     const int image_area = this->input_size * this->input_size;
  72.     this->input_image.resize(3 * image_area);
  73.     size_t single_chn_size = image_area * sizeof(float);
  74.     memcpy(this->input_image.data(), (float *)bgrChannels[0].data, single_chn_size);
  75.     memcpy(this->input_image.data() + image_area, (float *)bgrChannels[1].data, single_chn_size);
  76.     memcpy(this->input_image.data() + image_area * 2, (float *)bgrChannels[2].data, single_chn_size);
  77. }
  78.  
  79. void FaceAnalysis::generate_proposal(const float* p_box, const float* p_scores, const float* p_kps, const int stride, vector<Bbox>& boxes)
  80. {
  81. 	const int feat_h = this->input_size / stride;
  82. 	const int feat_w = this->input_size / stride;
  83.     const int num_anchors = 2;
  84. 	for (int i = 0; i < feat_h; i++)
  85. 	{
  86. 		for (int j = 0; j < feat_w; j++)
  87. 		{
  88. 			for(int n=0; n<num_anchors; n++)
  89.             {
  90.                 const int index = i * feat_w*num_anchors + j*num_anchors+n;
  91.                 if(p_scores[index] >= this->det_thresh)
  92.                 {
  93.                     Bbox box;
  94.                     box.xmin = (j - p_box[index * 4]) * stride;
  95.                     box.ymin = (i - p_box[index * 4 + 1]) * stride;
  96.                     box.xmax = (j + p_box[index * 4 + 2]) * stride;
  97.                     box.ymax = (i + p_box[index * 4 + 3]) * stride;
  98.                     box.xmin /= this->det_scale;
  99.                     box.ymin /= this->det_scale;
  100.                     box.xmax /= this->det_scale;
  101.                     box.ymax /= this->det_scale;
  102.  
  103.                     for(int k=0;k<5;k++)
  104.                     {
  105.                         float px = (j + p_kps[index * 10 + k * 2]) * stride;
  106.                         float py = (i + p_kps[index * 10 + k * 2 + 1]) * stride;
  107.                         px /= this->det_scale;
  108.                         py /= this->det_scale;
  109.                         box.kps[k * 2] = px;
  110.                         box.kps[k * 2 + 1] = py;
  111.                     }
  112.                     box.score = p_scores[index];
  113.                     boxes.emplace_back(box);
  114.                 }
  115.  
  116.             }
  117. 		}
  118. 	}
  119. }
  120.  
  121. bool cmp(Bbox a, Bbox b)
  122. {
  123.     float area_a = (a.xmax - a.xmin) * (a.ymax - a.ymin);
  124.     float area_b = (b.xmax - b.xmin) * (b.ymax - b.ymin);
  125.     return area_a > area_b;
  126. }
  127. vector<Bbox> FaceAnalysis::detect(const Mat& srcimg)
  128. {
  129.     this->preprocess(srcimg);
  130.  
  131.     std::vector<int64_t> input_img_shape = {1, 3, this->input_size, this->input_size};  ///也可以写在构造函数里, det_face的输入是动态的
  132.     Value input_tensor_ = Value::CreateTensor<float>(memory_info_handler, this->input_image.data(), this->input_image.size(), input_img_shape.data(), input_img_shape.size());
  133.  
  134.     vector<Value> det_face_ort_outputs = this->det_face_ort_session->Run(runOptions, this->det_face_input_names.data(), &input_tensor_, 1, this->det_face_output_names.data(), this->det_face_output_names.size());
  135.     vector<Bbox> boxes;
  136.     for(int i=0;i<3;i++)
  137.     {
  138.         float *p_scores = det_face_ort_outputs[i].GetTensorMutableData<float>();
  139.         float *p_bbox = det_face_ort_outputs[i + this->fmc].GetTensorMutableData<float>();
  140.         float *p_kps = det_face_ort_outputs[i + this->fmc*2].GetTensorMutableData<float>();
  141.         
  142.         this->generate_proposal(p_bbox, p_scores, p_kps, this->feat_stride_fpn[i], boxes);
  143.     }
  144.     nms_boxes(boxes, this->nms_thresh);
  145.  
  146.     for(int i=0;i<boxes.size();i++)
  147.     {
  148.         get_landmark
  149.         float w = boxes[i].xmax - boxes[i].xmin;
  150.         float h = boxes[i].ymax - boxes[i].ymin;
  151.         float center[2] = {(boxes[i].xmin + boxes[i].xmax) * 0.5f, (boxes[i].ymin + boxes[i].ymax) * 0.5f};
  152.         float rot = 0.f*PI/180.f;
  153.         float scale_ratio = this->landmark_input_size / (max(w, h) * 1.5);
  154.         face_align
  155.         Mat M = (Mat_<float>(2, 3) << scale_ratio*cos(rot), -scale_ratio*sin(rot), this->landmark_input_size*0.5-center[0]*scale_ratio, scale_ratio*sin(rot), scale_ratio*cos(rot), this->landmark_input_size*0.5-center[1]*scale_ratio);
  156.         Mat cropped;
  157. 	    warpAffine(srcimg, cropped, M, cv::Size(this->landmark_input_size, this->landmark_input_size));
  158.         face_align
  159.         vector<cv::Mat> bgrChannels(3);
  160.         split(cropped, bgrChannels);
  161.         for (int c = 0; c < 3; c++)
  162.         {
  163.             bgrChannels[c].convertTo(bgrChannels[c], CV_32FC1);
  164.         }
  165.  
  166.         const int image_area = this->landmark_input_size * this->landmark_input_size;
  167.         this->aimg.resize(3 * image_area);
  168.         size_t single_chn_size = image_area * sizeof(float);
  169.         memcpy(this->aimg.data(), (float *)bgrChannels[0].data, single_chn_size);
  170.         memcpy(this->aimg.data() + image_area, (float *)bgrChannels[1].data, single_chn_size);
  171.         memcpy(this->aimg.data() + image_area * 2, (float *)bgrChannels[2].data, single_chn_size);
  172.  
  173.         Value input_tensor2 = Value::CreateTensor<float>(memory_info_handler, this->aimg.data(), this->aimg.size(), this->landmark_input_tensor_shape.data(), this->landmark_input_tensor_shape.size());
  174.         
  175.         vector<Value> landmark_ort_outputs = this->landmark_ort_session->Run(runOptions, this->landmark_input_names.data(), &input_tensor2, 1, this->landmark_output_names.data(), this->landmark_output_names.size());
  176.         float *p_landmark = landmark_ort_outputs[0].GetTensorMutableData<float>();
  177.         Mat IM;
  178. 	    invertAffineTransform(M, IM);
  179.         for(int k=0;k<106;k++)
  180.         {
  181.             float px = (p_landmark[k * 2] + 1) * this->landmark_input_size*0.5;
  182.             float py = (p_landmark[k * 2 + 1] + 1) * this->landmark_input_size*0.5;
  183.             boxes[i].landmark_2d_106[k * 2] = IM.at<float>(0, 0) * px + IM.at<float>(0, 1) * py + IM.at<float>(0, 2);
  184.             boxes[i].landmark_2d_106[k * 2 + 1] = IM.at<float>(1, 0) * px + IM.at<float>(1, 1) * py + IM.at<float>(1, 2);
  185.         }
  186.         get_landmark
  187.     }
  188.     sort(boxes.begin(), boxes.end(), cmp);
  189.     return boxes;
  190. }

天天代码码天天
微信公众号
.NET 人工智能实践
注：本文转载自blog.csdn.net的天天代码码天天的文章"https://lw112190.blog.csdn.net/article/details/141061879"。版权归原作者所有，此博客不拥有其著作权，亦不承担相应法律责任。如有侵权，请联系我们删除。
复制链接
相关推荐
发表评论

评论记录：

未查询到任何数据！