为什么AI算法工程师要求C++？

pip install torchlm>=0.1.6.9  or install the latest pypi version `pip install torchlm`
pip install torchlm>=0.1.6.9 -i https://pypi.org/simple/  or install from specific pypimirrorsuse -i

--------------------------- 这部分是管理整理工程的代码 被下游的各个模块依赖 ----------------------------------
├── backend.h           宏处理 决定基础的推理引擎 目前必须是ONNXRuntime
├── config.h            宏处理
├── config.h.in         cmake编译时宏处理
├── lite.ai.defs.h      宏处理
├── lite.ai.headers.h   引入基础依赖库
├── lite.h              引入项目的基础对外的模块
├── pipeline            暂时没用
├── pipeline.h          暂时没用
├── types.h             基础类型，很重要，会被下游模块复用，比如实际上ort::types是types的alias
├── utils.cpp           基础功能函数实现，很重要，会被下游模块复用
└── utils.h             基础功能函数头文件
├── models.h            模型整体的命名空间管理，很重要，所有被实现的模型需要在这里被导出

 --------------------------- 以下的各个部分是相互独立的 但是需要依赖上面这个整体的部分 --------------------------
...
├── mnn
│   ├── core       MNN基础父类和特定功能的实现，必须要阅读
│   │   ├── mnn_config.h│   │   ├── mnn_core.h        MNN模型命名空间管理，实现一个类前，现在这里添加签名
│   │   ├── mnn_defs.h
│   │   ├── mnn_handler.cpp   基础父类实现，必须阅读
│   │   ├── mnn_handler.h
│   │   ├── mnn_types.h
│   │   ├── mnn_utils.cpp
│   │   └── mnn_utils.h
│   └── cv        各个模型的具体实现，会引用core中实现的父类和功能函数
│       ├── mnn_age_googlenet.cpp
│       ├── mnn_age_googlenet.h
│       ├── mnn_cava_combined_face.cpp
...
├── ncnn
│   ├── core       NCNN基础父类和特定功能的实现，必须要阅读
│   │   ├── ncnn_config.h
│   │   ├── ncnn_core.h         NCNN模型命名空间管理，实现一个类前，现在这里添加签名
│   │   ├── ncnn_custom.cpp
│   │   ├── ncnn_custom.h
│   │   ├── ncnn_defs.h
│   │   ├── ncnn_handler.cpp    基础父类实现，必须阅读
│   │   ├── ncnn_handler.h
│   │   ├── ncnn_types.h
│   │   ├── ncnn_utils.cpp
│   │   └── ncnn_utils.h
│   └── cv        各个模型的具体实现，会引用core中实现的父类和功能函数
│       ├── ncnn_age_googlenet.cpp
│       ├── ncnn_age_googlenet.h
│       ├── ncnn_cava_combined_face.cpp
│       ├── ncnn_cava_combined_face.h
│       ├── ncnn_cava_ghost_arcface.cpp
...
├── ort
│   ├── core       ONNXRuntime基础父类和特定功能的实现，必须要阅读
│   │   ├── ort_config.h
│   │   ├── ort_core.h         ONNXRuntime模型命名空间管理，实现一个类前，现在这里添加签名
│   │   ├── ort_defs.h
│   │   ├── ort_handler.cpp    基础父类实现，必须阅读
│   │   ├── ort_handler.h
│   │   ├── ort_types.h
│   │   ├── ort_utils.cpp
│   │   └── ort_utils.h
│   └── cv        各个模型的具体实现，会引用core中实现的父类和功能函数
│       ├── age_googlenet.cpp
│       ├── age_googlenet.h
│       ├── cava_combined_face.cpp
│       ├── cava_combined_face.h
│       ├── cava_ghost_arcface.cpp
│       ├── cava_ghost_arcface.h
...
├── tnn
│   ├── core       TNN基础父类和特定功能的实现，必须要阅读
│   │   ├── tnn_config.h
│   │   ├── tnn_core.h         TNN模型命名空间管理，实现一个类前，现在这里添加签名
│   │   ├── tnn_defs.h
│   │   ├── tnn_handler.cpp    基础父类实现，必须阅读
│   │   ├── tnn_handler.h
│   │   ├── tnn_types.h
│   │   ├── tnn_utils.cpp
│   │   └── tnn_utils.h
│   └── cv      各个模型的具体实现，会引用core中实现的父类和功能函数
│       ├── tnn_age_googlenet.cpp
│       ├── tnn_age_googlenet.h
│       ├── tnn_cava_combined_face.cpp
│       ├── tnn_cava_combined_face.h
│       ├── tnn_cava_ghost_arcface.cpp
│       ├── tnn_cava_ghost_arcface.h
│       ├── tnn_center_loss_face.cpp

// lite/ort/core/ort_core.h 中
namespace ortcv
{
  // ... 
  class LITE_EXPORTS YoloX;                      // [56] * reference: https://github.com/Megvii-BaseDetection/YOLOX
}
// lite/mnn/core/mnn_core.h 中
namespace mnncv
{
// ... 
class LITE_EXPORTS MNNYoloX;                      // [3] * reference: https://github.com/Megvii-BaseDetection/YOLOX
}

├── ort
│   ├── core       ONNXRuntime基础父类和特定功能的实现，必须要阅读
│   │   ├── ort_config.h
│...
│   │   └── ort_utils.h
│   └── cv        各个模型的具体实现，会引用core中实现的父类和功能函数
│       ├── yolox.cpp
│       ├── yolox.h

ifndef LITE_AI_ORT_CV_YOLOX_H
define LITE_AI_ORT_CV_YOLOX_H

include "lite/ort/core/ort_core.h"

namespace ortcv
{
  class LITE_EXPORTS YoloX : public BasicOrtHandler
  {
  public:
    explicit YoloX(const std::string &_onnx_path, unsigned int _num_threads = 1) :
        BasicOrtHandler(_onnx_path, _num_threads)
    {};

    ~YoloX() override = default;

  private:
    // nested classes
    typedef struct GridAndStride
    {
      int grid0;
      int grid1;
      int stride;
    } YoloXAnchor;

    typedef struct
    {
      float r;
      int dw;
      int dh;
      int new_unpad_w;
      int new_unpad_h;
      bool flag;
    } YoloXScaleParams;

  private:
    const float mean_vals[3] = {255.f * 0.485f, 255.f * 0.456, 255.f * 0.406f};
    const float scale_vals[3] = {1 / (255.f * 0.229f), 1 / (255.f * 0.224f), 1 / (255.f * 0.225f)};

    const char *class_names[80] = {
        "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
        "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
        "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
        "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
        "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
        "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
        "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard",
        "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
        "scissors", "teddy bear", "hair drier", "toothbrush"
    };
    enum NMS
    {
      HARD = 0, BLEND = 1, OFFSET = 2
    };
    static constexpr const unsigned int max_nms = 30000;

  private:
    // 需要被重写的方法
    Ort::Value transform(const cv::Mat &mat_rs) override; // without resize

    void resize_unscale(const cv::Mat &mat,
                        cv::Mat &mat_rs,
                        int target_height,
                        int target_width,
                        YoloXScaleParams &scale_params);

    void generate_anchors(const int target_height,
                          const int target_width,
                          std::vector &strides,
                          std::vector &anchors);

    void generate_bboxes(const YoloXScaleParams &scale_params,
                         std::vector &bbox_collection,
                         std::vector &output_tensors,
                         float score_threshold, int img_height,
                         int img_width); // rescale & exclude

    void nms(std::vector &input, std::vector &output,
             float iou_threshold, unsigned int topk, unsigned int nms_type);

  public:
    // 请保持detect、detect_video等的命名规范，detect是图片级别的检测接口，detect_video是视频级别的检测接口
    void detect(const cv::Mat &mat, std::vector &detected_boxes,
                float score_threshold = 0.25f, float iou_threshold = 0.45f,
                unsigned int topk = 100, unsigned int nms_type = NMS::OFFSET);

  };
}
endif //LITE_AI_ORT_CV_YOLOX_H

include "yolox.h"
include "lite/ort/core/ort_utils.h" // 引入onnxruntime特定的自定义功能函数，依赖于推理引擎
include "lite/utils.h" // 引入全局定义的功能函数，不依赖推理引擎，如NMS

using ortcv::YoloX;

Ort::Value YoloX::transform(const cv::Mat &mat_rs)
{
  cv::Mat canvas;
  cv::cvtColor(mat_rs, canvas, cv::COLOR_BGR2RGB);
  // resize without padding, (Done): add padding as the official Python implementation.
  // cv::resize(canva, canva, cv::Size(input_node_dims.at(3),
  //                                  input_node_dims.at(2)));
  // (1,3,640,640) 1xCXHXW
  ortcv::utils::transform::normalize_inplace(canvas, mean_vals, scale_vals); // float32
  // Note !!!: Comment out this line if you use the newest YOLOX model.
  // There is no normalization for the newest official C++ implementation
  // using ncnn. Reference:
  // [1] https://github.com/Megvii-BaseDetection/YOLOX/blob/main/demo/ncnn/cpp/yolox.cpp
  // ortcv::utils::transform::normalize_inplace(canva, mean_vals, scale_vals); // float32
  return ortcv::utils::transform::create_tensor(
      canvas, input_node_dims, memory_info_handler,
      input_values_handler, ortcv::utils::transform::CHW);
}

void YoloX::resize_unscale(const cv::Mat &mat, cv::Mat &mat_rs,
                           int target_height, int target_width,
                           YoloXScaleParams &scale_params)
{
  if (mat.empty()) return;
  int img_height = static_cast(mat.rows);
  int img_width = static_cast(mat.cols);

  mat_rs = cv::Mat(target_height, target_width, CV_8UC3,
                   cv::Scalar(114, 114, 114));
  // scale ratio (new / old) new_shape(h,w)
  float w_r = (float) target_width / (float) img_width;
  float h_r = (float) target_height / (float) img_height;
  float r = std::min(w_r, h_r);
  // compute padding
  int new_unpad_w = static_cast((float) img_width * r); // floor
  int new_unpad_h = static_cast((float) img_height * r); // floor
  int pad_w = target_width - new_unpad_w; // >=0
  int pad_h = target_height - new_unpad_h; // >=0

  int dw = pad_w / 2;
  int dh = pad_h / 2;

  // resize with unscaling
  cv::Mat new_unpad_mat = mat.clone();
  cv::resize(new_unpad_mat, new_unpad_mat, cv::Size(new_unpad_w, new_unpad_h));
  new_unpad_mat.copyTo(mat_rs(cv::Rect(dw, dh, new_unpad_w, new_unpad_h)));

  // record scale params.
  scale_params.r = r;
  scale_params.dw = dw;
  scale_params.dh = dh;
  scale_params.new_unpad_w = new_unpad_w;
  scale_params.new_unpad_h = new_unpad_h;
  scale_params.flag = true;
}

void YoloX::detect(const cv::Mat &mat, std::vector &detected_boxes,
                   float score_threshold, float iou_threshold,
                   unsigned int topk, unsigned int nms_type)
{
  if (mat.empty()) return;
  const int input_height = input_node_dims.at(2);
  const int input_width = input_node_dims.at(3);
  int img_height = static_cast(mat.rows);
  int img_width = static_cast(mat.cols);

  // resize & unscale
  cv::Mat mat_rs;
  YoloXScaleParams scale_params;
  this->resize_unscale(mat, mat_rs, input_height, input_width, scale_params);

  // 1. make input tensor
  Ort::Value input_tensor = this->transform(mat_rs);
  // 2. inference scores & boxes.
  auto output_tensors = ort_session->Run(
      Ort::RunOptions{nullptr}, input_node_names.data(),
      &input_tensor, 1, output_node_names.data(), num_outputs
  );
  // 3. rescale & exclude.
  std::vector bbox_collection;
  this->generate_bboxes(scale_params, bbox_collection, output_tensors, score_threshold, img_height, img_width);
  // 4. hard|blend|offset nms with topk.
  this->nms(bbox_collection, detected_boxes, iou_threshold, topk, nms_type);
}

void YoloX::generate_anchors(const int target_height,
                             const int target_width,
                             std::vector &strides,
                             std::vector &anchors)
{
  for (auto stride : strides)
  {
    int num_grid_w = target_width / stride;
    int num_grid_h = target_height / stride;
    for (int g1 = 0; g1 < num_grid_h; ++g1)
    {
      for (int g0 = 0; g0 < num_grid_w; ++g0)
      {
ifdef LITE_WIN32
        YoloXAnchor anchor;
        anchor.grid0 = g0;
        anchor.grid1 = g1;
        anchor.stride = stride;
        anchors.push_back(anchor);
else
        anchors.push_back((YoloXAnchor) {g0, g1, stride});
endif
      }
    }
  }
}

void YoloX::generate_bboxes(const YoloXScaleParams &scale_params,
                            std::vector &bbox_collection,
                            std::vector &output_tensors,
                            float score_threshold, int img_height,
                            int img_width)
{
  Ort::Value &pred = output_tensors.at(0); // (1,n,85=5+80=cxcy+cwch+obj_conf+cls_conf)
  auto pred_dims = output_node_dims.at(0); // (1,n,85)
  const unsigned int num_anchors = pred_dims.at(1); // n = ?
  const unsigned int num_classes = pred_dims.at(2) - 5;
  const float input_height = static_cast(input_node_dims.at(2)); // e.g 640
  const float input_width = static_cast(input_node_dims.at(3)); // e.g 640

  std::vector anchors;
  std::vector strides = {8, 16, 32}; // might have stride=64
  this->generate_anchors(input_height, input_width, strides, anchors);

  float r_ = scale_params.r;
  int dw_ = scale_params.dw;
  int dh_ = scale_params.dh;

  bbox_collection.clear();
  unsigned int count = 0;
  for (unsigned int i = 0; i < num_anchors; ++i)
  {
    float obj_conf = pred.At({0, i, 4});
    if (obj_conf < score_threshold) continue; // filter first.

    float cls_conf = pred.At({0, i, 5});
    unsigned int label = 0;
    for (unsigned int j = 0; j < num_classes; ++j)
    {
      float tmp_conf = pred.At({0, i, j + 5});
      if (tmp_conf > cls_conf)
      {
        cls_conf = tmp_conf;
        label = j;
      }
    } // argmax
    float conf = obj_conf * cls_conf; // cls_conf (0.,1.)
    if (conf < score_threshold) continue; // filter

    const int grid0 = anchors.at(i).grid0;
    const int grid1 = anchors.at(i).grid1;
    const int stride = anchors.at(i).stride;

    float dx = pred.At({0, i, 0});
    float dy = pred.At({0, i, 1});
    float dw = pred.At({0, i, 2});
    float dh = pred.At({0, i, 3});

    float cx = (dx + (float) grid0) * (float) stride;
    float cy = (dy + (float) grid1) * (float) stride;
    float w = std::exp(dw) * (float) stride;
    float h = std::exp(dh) * (float) stride;
    float x1 = ((cx - w / 2.f) - (float) dw_) / r_;
    float y1 = ((cy - h / 2.f) - (float) dh_) / r_;
    float x2 = ((cx + w / 2.f) - (float) dw_) / r_;
    float y2 = ((cy + h / 2.f) - (float) dh_) / r_;

    types::Boxf box;
    box.x1 = std::max(0.f, x1);
    box.y1 = std::max(0.f, y1);
    box.x2 = std::min(x2, (float) img_width - 1.0f);
    box.y2 = std::min(y2, (float) img_height - 1.0f);
    box.score = conf;
    box.label = label;
    box.label_text = class_names[label];
    box.flag = true;
    bbox_collection.push_back(box);

    count += 1; // limit boxes for nms.
    if (count > max_nms)
      break;
  }
if LITEORT_DEBUG
  std::cout << "detected num_anchors: " << num_anchors << "\n";
  std::cout << "generate_bboxes num: " << bbox_collection.size() << "\n";
endif
}

void YoloX::nms(std::vector &input, std::vector &output,
                float iou_threshold, unsigned int topk, unsigned int nms_type)
{
  if (nms_type == NMS::BLEND) lite::utils::blending_nms(input, output, iou_threshold, topk);
  else if (nms_type == NMS::OFFSET) lite::utils::offset_nms(input, output, iou_threshold, topk);
  else lite::utils::hard_nms(input, output, iou_threshold, topk);
}

// ENABLE_ONNXRUNTIME
ifdef ENABLE_ONNXRUNTIME
// ...
include "lite/ort/cv/yolox.h"
endif

// 默认版本
namespace lite 
{
  namespace cv 
  {
ifdef BACKEND_ONNXRUNTIME
    typedef ortcv::YoloX _YoloX;
endif
  }
    // 2. general object detection
  namespace detection
  {
ifdef BACKEND_ONNXRUNTIME
    typedef _YoloX YoloX;
endif
  }
}
// 还有个onnxruntime的命名空间也要添加
namespace lite
{
  namespace onnxruntime 
  {
    namespace cv
    {
      typedef ortcv::YoloX _ONNXYoloX;
    }
    // 2. general object detection
    namespace detection
    {
      typedef _ONNXYoloX YoloX;
    }
  }
}

include "lite/lite.h"

static void test_default()
{
  std::string onnx_path = "../../../hub/onnx/cv/yolox_s.onnx";
  std::string test_img_path = "../../../examples/lite/resources/test_lite_yolox_1.jpg";
  std::string save_img_path = "../../../logs/test_lite_yolox_1.jpg";

  // 1. Test Default Engine ONNXRuntime
  lite::cv::detection::YoloX *yolox = new lite::cv::detection::YoloX(onnx_path); // default

  std::vector detected_boxes;
  cv::Mat img_bgr = cv::imread(test_img_path);
  yolox->detect(img_bgr, detected_boxes);

  lite::utils::draw_boxes_inplace(img_bgr, detected_boxes);

  cv::imwrite(save_img_path, img_bgr);

  std::cout << "Default Version Detected Boxes Num: " << detected_boxes.size() << std::endl;

  delete yolox;

}

static void test_onnxruntime()
{
ifdef ENABLE_ONNXRUNTIME
  std::string onnx_path = "../../../hub/onnx/cv/yolox_s.onnx";
  std::string test_img_path = "../../../examples/lite/resources/test_lite_yolox_2.jpg";
  std::string save_img_path = "../../../logs/test_lite_yolox_2.jpg";

  // 2. Test Specific Engine ONNXRuntime
  lite::onnxruntime::cv::detection::YoloX *yolox =
      new lite::onnxruntime::cv::detection::YoloX(onnx_path);

  std::vector detected_boxes;
  cv::Mat img_bgr = cv::imread(test_img_path);
  yolox->detect(img_bgr, detected_boxes);

  lite::utils::draw_boxes_inplace(img_bgr, detected_boxes);

  cv::imwrite(save_img_path, img_bgr);

  std::cout << "ONNXRuntime Version Detected Boxes Num: " << detected_boxes.size() << std::endl;

  delete yolox;
endif
}

static void test_mnn()
{
ifdef ENABLE_MNN
  // ...
endif
}

static void test_ncnn()
{
ifdef ENABLE_NCNN
  // ...
endif
}

static void test_tnn()
{
ifdef ENABLE_TNN
  // ...
endif
}

static void test_lite()
{
  test_default();
  test_onnxruntime();
  test_mnn();
  test_ncnn();
  test_tnn();
}

int main(__unused int argc, __unused char *argv[])
{
  test_lite();
  return 0;
}

...
add_lite_executable(lite_yolox cv)

test_lite_xxx.cpp  添加到CMakeLists时，使用add_lite_executable(lite_xxx, cv)

sh ./build.sh && cd build/lite.ai.toolkit/bin && ./lite_yolox

PP-YOLOE的部署 
import fastdeploy as fd
import cv2
model = fd.vision.detection.PPYOLOE("model.pdmodel", 
                                    "model.pdiparams", 
                                     "infer_cfg.yml")
im = cv2.imread("test.jpg")
result = model.predict(im) YOLOv7的部署
import fastdeploy as fd
import cv2
model = fd.vision.detection.YOLOv7("model.onnx")
im = cv2.imread("test.jpg")
result = model.predict(im)

PP-YOLOE的部署
import fastdeploy as fd
import cv2
option = fd.RuntimeOption()
option.use_cpu()
option.use_openvino_backend()  一行命令切换使用 OpenVINO部署
model = fd.vision.detection.PPYOLOE("model.pdmodel", 
                                    "model.pdiparams", 
                                    "infer_cfg.yml",
                                    runtime_option=option)
im = cv2.imread("test.jpg")
result = model.predict(im)

pip install fastdeploy-gpu-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html
git clone https://github.com/PaddlePaddle/FastDeploy.git
cd examples/vision/detection/yolov7/python/

wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov7.onnx
wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg

CPU推理
python infer.py --model yolov7.onnx --image 000000014439.jpg --device cpu
 GPU推理
python infer.py --model yolov7.onnx --image 000000014439.jpg --device gpu
 GPU上使用TensorRT推理
python infer.py --model yolov7.onnx --image 000000014439.jpg --device gpu --use_trt True

git clone https://github.com/PaddlePaddle/FastDeploy cd FastDeploy
mkdir build && cd build
cmake .. ­DBUILD_ON_JETSON=ON ­DENABLE_VISION=ON ­DCMAKE_INSTALL_PREFIX=${PWD}/install make ­j8
make install
cd FastDeploy/build/install
source fastdeploy_init.sh

wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov7.onnx
wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg

cd examples/vision/detection/yolov7/cpp
cmake .. ­DFASTDEPLOY_INSTALL_DIR=${FASTDEPOLY_DIR} 
mkdir build && cd build
make ­j

 使用TensorRT推理(当模型不支持TensorRT时会自动转成使用CPU推理)
./infer_demo yolov7s.onnx 000000014439.jpg 2

参考编译文档，完成FastDeploy编译安装
 参考文档链接：https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/cn/build_and_install/rknpu2.md
 下载部署示例代码
git clone https://github.com/PaddlePaddle/FastDeploy.git
cd examples/vision/detection/paddledetection/rknpu2/python

wget https://bj.bcebos.com/fastdeploy/models/rknn2/picodet_s_416_coco_npu.zip
unzip -qo picodet_s_416_coco_npu.zip
 下载Paddle静态图模型并解压
wget https://bj.bcebos.com/fastdeploy/models/rknn2/picodet_s_416_coco_npu.zip
unzip -qo picodet_s_416_coco_npu.zip
 静态图转ONNX模型，注意，这里的save_file请和压缩包名对齐
paddle2onnx --model_dir picodet_s_416_coco_npu \
 --model_filename model.pdmodel \
 --params_filename model.pdiparams \
 --save_file picodet_s_416_coco_npu/picodet_s_416_coco_npu.onnx \
 --enable_dev_version True

python -m paddle2onnx.optimize --input_model picodet_s_416_coco_npu/picodet_s_416_coco_npu.onnx \
 --output_model picodet_s_416_coco_npu/picodet_s_416_coco_npu.onnx \
 --input_shape_dict "{image:[1,3,416,416]}"
 ONNX模型转RKNN模型
 转换模型,模型将生成在picodet_s_320_coco_lcnet_non_postprocess目录下
python tools/rknpu2/export.py --config_path tools/rknpu2/config/RK3588/picodet_s_416_coco_npu.yaml

 下载图片
wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg

python3 infer.py --model_file ./picodet _3588/picodet_3588.rknn \
 --config_file ./picodet_3588/deploy.yaml \
 --image images/000000014439.jpg