#include "VPTProcess.h" #include "MutliSourceVideoProcess.h" #ifndef _MSC_VER #include #endif #include #include #include "ObjCls.h" #include #include "ErrorInfo.h" #include "fstream" #define USE_YOLOV5 // #include "../../model/vptModeTrt/ga_vpt_init_net.h" // #include "../../model/vptModeTrt/ga_vpt_predict_net.h" // //#include "../../model/hidemodel_caffe_1108/ga_vpt_init_net_caffe2.h" // //#include "../../model/hidemodel_caffe_1108/ga_vpt_predict_net_caffe2.h" //model trt onnx 20210715 #include "../../model/vptModeTrt/ga_vpt_det_yolo_640x640.h" // debug by zsh #include "../../model/vptModeTrt/ga_trt_yolo_vpt_calibrator.h" // debug by zsh // // fpn // #include "vpt_fpn_plugin_factory.h" // #include "../../model/vptModeTrt/ga_trt_fpn_vpt_calibrator.h" typedef struct objDetector { void* det_handle; float threshold; int engine_type; // VPT_FPNPluginFactory tensorrt_plugin_factory; int licence_status = -1; int thrd_status = -1; vector taskTrackers; objDetector() { det_handle = NULL; threshold = 0.6; } }objDetector; static long long get_cur_time_ms(){ chrono::time_point tpMicro = chrono::time_point_cast(chrono::system_clock::now()); return tpMicro.time_since_epoch().count(); } void config_fpn_param(ctools_init_params& param, int batch_size) { // param.model_type_ = MODEL_FPN; // param.weight_file_ = NULL; // param.net_file_ = NULL; // param.data_process_str_ = // //"CopyData_CPU2GPU_U8;" // "TypeConvert_U8_F32;" // "ResizeMaxPad_F32_F32,test_size,720,test_max_size,1280,max_height,736,max_width,1280," // "submean_b,103.94,submean_g,116.78,submean_r,123.68," // "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" // "NHWC2NCHW_F32" // ; // param.need_im_info_ = 1; // true // if (param.engine_type_ == ENGINE_MCAFFE2) // { // /*param.weight_array_ = (unsigned char*)ga_vpt_init_net_caffe2; // param.weight_array_len_ = ga_vpt_init_net_len_caffe2; // param.net_array_ = (unsigned char*)ga_vpt_predict_net_caffe2; // param.net_array_len_ = ga_vpt_predict_net_len_caffe2;*/ // } // else if (param.engine_type_ == ENGINE_TENSORRT) // { // param.weight_array_ = (uint8_t*)ga_vpt_init_net; // param.weight_array_len_ = ga_vpt_init_net_len; // param.net_array_ = (uint8_t*)ga_vpt_predict_net; // param.net_array_len_ = ga_vpt_predict_net_len; // memset(param.tensorrt_param_str_, 0, sizeof(param.tensorrt_param_str_)); // std::string g_data_mode = "FP32"; // bool g_is_create_calibrator = false; // sprintf(param.tensorrt_param_str_, "max_batchsize %d," // "data_mode %s," // "is_create_calibrator %d," // "input_names data im_info," // "output_names cls_prob bbox_pred_final rois_count_each_img", // batch_size, g_data_mode.c_str(), g_is_create_calibrator); // param.tensorrt_calibrator_file_ = NULL;// "trt_fpn_vpt_calibrator"; // param.tensorrt_calibrator_array_len_ = ga_trt_fpn_vpt_calibrator_len;// "trt_fpn_vpt_calibrator"; // param.tensorrt_calibrator_array_ = (unsigned char*)ga_trt_fpn_vpt_calibrator;// "trt_fpn_vpt_calibrator"; // param.tensorrt_plugin_factory_ptr_ = &(tools->tensorrt_plugin_factory); // } } void config_yolo_param(ctools_init_params& param, int batch_size) { param.model_type_ = MODEL_YOLOV5; // debug by zsh param.weight_file_ = NULL; param.net_file_ = NULL; param.data_process_str_ = //"CopyData_CPU2GPU_U8;" "TypeConvert_U8_F32;" "ResizeMaxMidPad_F32_F32,test_size,640,test_max_size,640,max_height,640,max_width,640," "submean_b,0,submean_g,0,submean_r,0," "variance_rev_b,0.00392,variance_rev_g,0.00392,variance_rev_r,0.00392;" "BGR2RGB_F32_F32;" "NHWC2NCHW_F32" ; param.need_im_info_ = 0; if (param.engine_type_ == ENGINE_MCAFFE2) { /*param.weight_array_ = (unsigned char*)ga_vpt_init_net_caffe2; param.weight_array_len_ = ga_vpt_init_net_len_caffe2; param.net_array_ = (unsigned char*)ga_vpt_predict_net_caffe2; param.net_array_len_ = ga_vpt_predict_net_len_caffe2;*/ } else if (param.engine_type_ == ENGINE_TENSORRT) { param.net_array_ = (uint8_t*)ga_vpt_det_yolo_640x640; param.net_array_len_ = ga_vpt_det_yolo_640x640_len; //debug by zsh memset(param.tensorrt_param_str_, 0, sizeof(param.tensorrt_param_str_)); std::string g_data_mode = "FP32"; bool g_is_create_calibrator = false; int g_is_onnx_model = 1; sprintf(param.tensorrt_param_str_, "max_batchsize %d," "data_mode %s," "is_create_calibrator %d," "is_onnx_model %d," "input_names images," "output_names output", batch_size, g_data_mode.c_str(), g_is_create_calibrator, g_is_onnx_model); param.tensorrt_calibrator_array_len_ = ga_trt_yolo_vpt_calibrator_len;// param.tensorrt_calibrator_array_ = (unsigned char*)ga_trt_yolo_vpt_calibrator;// debug by zsh // param.tensorrt_plugin_factory_ptr_ = &(tools->tensorrt_plugin_factory); } } int VPT_Init(void *&handle, VPTProcess_PARAM vparam) { objDetector *tools = new objDetector; ctools_init_params param; param.thres_ = vparam.threshold; param.log_level_ = 0; param.device_type_ = vparam.mode; param.device_id_ = vparam.gpuid; param.engine_type_ = vparam.engine; param.trt_serialize_file_ = vparam.serialize_file; #ifdef USE_YOLOV5 config_yolo_param(param, vparam.max_batch); #else config_fpn_param(param, vparam.max_batch); #endif tools->threshold = vparam.threshold; tools->engine_type = vparam.engine; // debug by zsh int flag = ctools_init(&(tools->det_handle), ¶m); if (SUCCESS != flag) { if (tools) { delete tools; tools = NULL; } } else { handle = tools; } return flag; } // int VPT_Init(void *&handle, VPTProcess_PARAM vparam) // { // objDetector *tools = new objDetector; // ctools_init_params param; // param.thres_ = vparam.threshold; // param.log_level_ = 0; // param.device_type_ = vparam.mode; // param.device_id_ = vparam.gpuid; // param.engine_type_ = vparam.engine; // param.trt_serialize_file_ = vparam.serialize_file; // param.model_type_ = MODEL_FPN; // // param.model_type_ = MODEL_YOLOV5; // debug by zsh // param.weight_file_ = NULL; // param.net_file_ = NULL; // param.data_process_str_ = // //"CopyData_CPU2GPU_U8;" // "TypeConvert_U8_F32;" // "ResizeMaxPad_F32_F32,test_size,720,test_max_size,1280,max_height,736,max_width,1280," // "submean_b,103.94,submean_g,116.78,submean_r,123.68," // "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" // "NHWC2NCHW_F32" // ; // param.need_im_info_ = 1; // true // // param.data_process_str_ = // // //"CopyData_CPU2GPU_U8;" // // "TypeConvert_U8_F32;" // // "ResizeMaxMidPad_F32_F32,test_size,640,test_max_size,640,max_height,640,max_width,640," // // "submean_b,0,submean_g,0,submean_r,0," // // "variance_rev_b,0.00392,variance_rev_g,0.00392,variance_rev_r,0.00392;" // // "BGR2RGB_F32_F32;" // // "NHWC2NCHW_F32" // // ; // // param.need_im_info_ = 0; // if (param.engine_type_ == ENGINE_MCAFFE2) // { // /*param.weight_array_ = (unsigned char*)ga_vpt_init_net_caffe2; // param.weight_array_len_ = ga_vpt_init_net_len_caffe2; // param.net_array_ = (unsigned char*)ga_vpt_predict_net_caffe2; // param.net_array_len_ = ga_vpt_predict_net_len_caffe2;*/ // } // else if (param.engine_type_ == ENGINE_TENSORRT) // { // param.weight_array_ = (uint8_t*)ga_vpt_init_net; // param.weight_array_len_ = ga_vpt_init_net_len; // param.net_array_ = (uint8_t*)ga_vpt_predict_net; // param.net_array_len_ = ga_vpt_predict_net_len; // // param.net_array_ = (uint8_t*)ga_vpt_det_yolo_640x640; // // param.net_array_len_ = ga_vpt_det_yolo_640x640_len; //debug by zsh // param.trt_serialize_file_ = vparam.serialize_file; // memset(param.tensorrt_param_str_, 0, sizeof(param.tensorrt_param_str_)); // int batch_size = vparam.max_batch; // std::string g_data_mode = "FP32"; // bool g_is_create_calibrator = false; // // int g_is_onnx_model = 1; // // sprintf(param.tensorrt_param_str_, "max_batchsize %d," // // "data_mode %s," // // "is_create_calibrator %d," // // "is_onnx_model %d," // // "input_names images," // // "output_names output", // // batch_size, g_data_mode.c_str(), g_is_create_calibrator, g_is_onnx_model); // sprintf(param.tensorrt_param_str_, "max_batchsize %d," // "data_mode %s," // "is_create_calibrator %d," // "input_names data im_info," // "output_names cls_prob bbox_pred_final rois_count_each_img", // batch_size, g_data_mode.c_str(), g_is_create_calibrator); // param.tensorrt_calibrator_file_ = NULL;// "trt_fpn_vpt_calibrator"; // param.tensorrt_calibrator_array_len_ = ga_trt_fpn_vpt_calibrator_len;// "trt_fpn_vpt_calibrator"; // param.tensorrt_calibrator_array_ = (unsigned char*)ga_trt_fpn_vpt_calibrator;// "trt_fpn_vpt_calibrator"; // // param.tensorrt_calibrator_array_len_ = ga_trt_yolo_vpt_calibrator_len;// // // param.tensorrt_calibrator_array_ = (unsigned char*)ga_trt_yolo_vpt_calibrator;// debug by zsh // param.tensorrt_plugin_factory_ptr_ = &(tools->tensorrt_plugin_factory); // } // tools->threshold = vparam.threshold; // tools->engine_type = vparam.engine; // debug by zsh // int flag = ctools_init(&(tools->det_handle), ¶m); // if (SUCCESS != flag) // { // if (tools) // { // delete tools; // tools = NULL; // } // } // else // { // handle = tools; // } // return flag; // } void check_VPT_Result(VPT_Result & vResult) { int index = 0; for (int i = 0; i < vResult.objCount; i++) { if ((vResult.obj[i].right - vResult.obj[i].left) > 10 && (vResult.obj[i].bottom - vResult.obj[i].top) > 10) { if (index == i) { index++; continue; } vResult.obj[index].left = vResult.obj[i].left; vResult.obj[index].top = vResult.obj[i].top; vResult.obj[index].right = vResult.obj[i].right; vResult.obj[index].bottom = vResult.obj[i].bottom; vResult.obj[index].center_x = vResult.obj[i].center_x; vResult.obj[index].center_y = vResult.obj[i].center_y; vResult.obj[index].index = vResult.obj[i].index; vResult.obj[index].id = vResult.obj[i].id; vResult.obj[index].num = vResult.obj[i].num; vResult.obj[index].confidence = vResult.obj[i].confidence; vResult.obj[index].snap_flag = vResult.obj[i].snap_flag; index++; } } vResult.objCount = index; } int VPT_Process_GPU(void * handle, sy_img * batch_img, int startBatch, int batchsize, vector vec_frameIndex, vector& result, vector>& deleteObjectID, vector>& unUsedResult) { objDetector *tools = (objDetector*)handle; bool isUseDet = true; int channels = 3; // if(batchsize == 5) { // for (int i = 0; i < batchsize; i++) { // // test // printf("k值: %d \n", i); // int height = batch_img[i].h_; // int width = batch_img[i].w_; // int rgb_size = 3 * width * height; // unsigned char *cpu_data = new unsigned char[rgb_size]; // cudaError_t cudaStatus = cudaMemcpy(cpu_data, batch_img[i].data_, rgb_size * sizeof(unsigned char), cudaMemcpyDeviceToHost); // cv::Mat img(height, width, CV_8UC3, cpu_data); // string filename = "test" + to_string(i) + ".jpg"; // cv::imwrite(filename.c_str(), img); // delete[] cpu_data; // cpu_data = nullptr; // } // } long long t1 = get_cur_time_ms(); ctools_result *detresult; int res_status = ctools_process(tools->det_handle, batch_img, batchsize, &detresult); #ifdef LOG_INFO2 long long t2 = get_cur_time_ms(); cout << "ctools_process time_using = "<< t2 - t1 << endl; #endif vector >> detectResult(batchsize); //sort for (int b = 0; b < batchsize; b++) { ctools_result &cur_result = detresult[b]; for (int c = 0; c < cur_result.obj_count_ && c < MAX_OBJ_COUNT; c++) { float x1 = cur_result.obj_results_[c].data_[2]; float y1 = cur_result.obj_results_[c].data_[3]; float x2 = cur_result.obj_results_[c].data_[4]; float y2 = cur_result.obj_results_[c].data_[5]; float class_id = cur_result.obj_results_[c].data_[0]; float score = cur_result.obj_results_[c].data_[1]; int imgid = b; if (score >= THRESHOLD) { vector obj; obj.push_back(x1); obj.push_back(y1); obj.push_back(x2); obj.push_back(y2); obj.push_back(score); obj.push_back(class_id); detectResult[imgid].push_back(obj); } } } int resIndex = startBatch; int detectIndex = 0; for (int i = startBatch; i < tools->taskTrackers.size(); i++) { TaskTracker task_tracker = tools->taskTrackers[i]; if (!task_tracker.tracker.GetState()) { continue; } if (task_tracker.lastFrameIndex <= 0) { // 第一帧,只做带检测框的跟踪 int objCount = task_tracker.tracker.update(task_tracker.ratioWidth, task_tracker.ratioHeight, true, detectResult[detectIndex], result[resIndex].obj, deleteObjectID[detectIndex]); result[resIndex].objCount = objCount; vector>().swap(detectResult[detectIndex]); detectResult[detectIndex].clear(); task_tracker.lastFrameIndex = vec_frameIndex[detectIndex]; continue; } int update_times = vec_frameIndex[detectIndex] - task_tracker.lastFrameIndex - 1; if (update_times < 0) { cout << "FrameIndex error !! lastFrameIndex= "<< task_tracker.lastFrameIndex <<" cur_frameindex = " << vec_frameIndex[detectIndex] << endl; } // cout << "lastFrameIndex= " << task_tracker.lastFrameIndex << " cur_frameindex = " << vec_frameIndex[detectIndex] << endl; // cout << "update_times = " << update_times << endl; for (int j = 0; j < update_times; j++) { // 无检测框跟踪 //cout << "taskTrackers size: " << task_tracker.size() << " batchsize:" << detectResult.size() << " update_times: "<< update_times << endl; VPT_Result unresult; VPT_ObjInfo obj[MAX_OBJ_COUNT]; unresult.objCount =task_tracker.tracker.update(task_tracker.ratioWidth, task_tracker.ratioHeight, false, detectResult[detectIndex], unresult.obj, deleteObjectID[detectIndex]); check_VPT_Result(unresult); unUsedResult[resIndex].push_back(unresult); } //有检测框输入的跟踪 int objCount = task_tracker.tracker.update(task_tracker.ratioWidth, task_tracker.ratioHeight, true, detectResult[detectIndex], result[resIndex].obj, deleteObjectID[detectIndex]); result[resIndex].objCount = objCount; vector>().swap(detectResult[detectIndex]); detectResult[detectIndex].clear(); // 记录帧序号 task_tracker.lastFrameIndex = vec_frameIndex[detectIndex]; check_VPT_Result(result[resIndex]); resIndex++; detectIndex++; if (resIndex == startBatch + batchsize) break; } #ifdef LOG_INFO2 long long t3 = get_cur_time_ms(); cout << "track time_using = "<< t3 - t2 << endl; #endif vector >>().swap(detectResult); return SUCCESS; } void VPT_Release(void * handle) { objDetector *tools = (objDetector*)handle; if (tools) { if (tools->det_handle) { ctools_release(&tools->det_handle); tools->det_handle = NULL; } vector().swap(tools->taskTrackers); delete tools; tools = NULL; } } void AddTaskTracker(void * handle, const int taskID, const double rWidth, const double rHeight) { objDetector *tools = (objDetector*)handle; TaskTracker t; t.TaskID = taskID; t.ratioWidth = rWidth; t.ratioHeight = rHeight; t.lastFrameIndex = 0; t.tracker.setYOLOv5(true); // YOLOv5 要设为true, fpn 要设为false tools->taskTrackers.push_back(t); } void FinishTaskTracker(void * handle, const int taskID) { objDetector *tools = (objDetector*)handle; for (int i = 0; i < tools->taskTrackers.size(); i++) { if (tools->taskTrackers[i].TaskID == taskID) { tools->taskTrackers.erase(tools->taskTrackers.begin() + i); break; } } } void PauseTaskTracker(void * handle, const int taskID) { objDetector *tools = (objDetector*)handle; for (int i = 0; i < tools->taskTrackers.size(); i++) { if (tools->taskTrackers[i].TaskID == taskID) { tools->taskTrackers[i].tracker.Pause(); break; } } } void RestartTaskTraker(void * handle, const int taskID) { objDetector *tools = (objDetector*)handle; for (int i = 0; i < tools->taskTrackers.size(); i++) { if (tools->taskTrackers[i].TaskID == taskID) { tools->taskTrackers[i].tracker.ReSet(); break; } } } void DrawTracker(void * handle, const int taskID, cv::Mat *img) { objDetector *tools = (objDetector*)handle; for (int i = 0; i < tools->taskTrackers.size(); i++) { if (tools->taskTrackers[i].TaskID == taskID) { tools->taskTrackers[i].tracker.addTracker(img); break; } } } int VPT_Process(void * handle, unsigned char ** bgr, int batchsize, VPT_Result *result) { // isUseDet = false; // } // } // resIndex++; // if (resIndex == batchsize) break; //} //vector >>().swap(detectResult); //vector>().swap(ssdResult); ///*detectResult.clear(); //ssdResult.clear();*/ return SUCCESS; } //#include void permute(float * image, int testWidth, int testHeight) { //cv::Mat host_image; float * host_image; //host_image.create(testHeight, testWidth, CV_32FC3); host_image = (float *)malloc(testHeight*testWidth * 3 * sizeof(float));; float *Host_img = new float[3 * testWidth * testHeight]{};//?????ڴ? float* image_data_original = image; //NPP_CHECK_CUDA(cudaMemcpy(Host_img, image_data_original, testWidth*testHeight * 3 * sizeof(float), cudaMemcpyDeviceToHost));//?????Կ???????ͼ???????? cudaMemcpy(Host_img, image_data_original, testWidth*testHeight * 3 * sizeof(float), cudaMemcpyDeviceToHost);//?????Կ???????ͼ???????? for (int j = 0; j < testHeight; j++) { float *pts = host_image + j * testWidth * 3; for (int i = 0; i < testWidth; i++) { //pts[3 * i] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 0]); //b //pts[3 * i + 1] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 1]); //g //pts[3 * i + 2] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 2]); //r pts[3 * i] = (Host_img[j * testWidth + i]); //b pts[3 * i + 1] = (Host_img[testWidth * testHeight + j * testWidth + i]); //g pts[3 * i + 2] = (Host_img[2 * testWidth * testHeight + j * testWidth + i]); //r } } cudaMemcpy(image_data_original, host_image, testWidth*testHeight * 3 * sizeof(float), cudaMemcpyHostToDevice); free(host_image); //cv::Mat showImg; //cv::resize(host_image, showImg, cv::Size(640, 480)); //cv::imshow("image", showImg); //cv::waitKey(0); } cv::Mat GpuMat2OpencvMat(unsigned char* image, int width, int height) { int testWidth = width; int testHeight = height; cv::Mat host_image; host_image.create(testHeight, testWidth, CV_8UC3); unsigned char *Host_img = new unsigned char[3 * testWidth * testHeight]{};//?????ڴ? unsigned char* image_data_original = image; cudaError_t code = cudaMemcpy(Host_img, image_data_original, testWidth*testHeight * 3 * sizeof(unsigned char), cudaMemcpyDeviceToHost);//?????Կ???????ͼ???????? if (code != 0) { printf("==========================================================error"); } std::ofstream outfile("decode.bin", ios::out | ios::binary); outfile.write((char*)Host_img, int(sizeof(char) * 1080 * 1920 * 3)); outfile.close(); cudaMemcpy(host_image.data, image_data_original, 1920 * testHeight * 3 * sizeof(unsigned char), cudaMemcpyDeviceToHost); cv::imwrite("input3.jpg", host_image); return host_image; } int VPT_ProcessImage(void * handle, unsigned char ** bgr, int batchsize, VPT_Result * result) { //objDetector* tools = (objDetector*)handle; //if (bgr == NULL) //ͼ?????ݴ??? // return IMG_DATA_ERROR; //vector imgs; ////int datasize = tools->param.w * tools->param.h * tools->param.c; //for (int i = 0; i < batchsize; i++) //{ // Mat img(tools->param.h, tools->param.w, tools->param.c, bgr[i]); // imgs.push_back(img); //} //vector> ssdResult; ////clock_t begin = clock(); //ssdResult = SSD_Detect(tools->detector, batchsize, imgs); // //vector >> detectResult(batchsize); //ת??Ϊ????????Ҫ?????? //for (int i = 0; i < ssdResult.size(); ++i) //{ // const vector& d = ssdResult[i];// Detection format: [image_id, label, score, xmin, ymin, xmax, ymax) // const float score = d[2]; // int imgid = d[0]; // if (score >= THRESHOLD) //????Ϊleft top right bottom score id // { // vector obj; // obj.push_back(d[3] * imgs[d[0]].cols); // obj.push_back(d[4] * imgs[d[0]].rows); // obj.push_back(d[5] * imgs[d[0]].cols); // obj.push_back(d[6] * imgs[d[0]].rows); // obj.push_back(d[2]); // obj.push_back(d[1]); // detectResult[imgid].push_back(obj); // } //} //for (int i = 0; i < batchsize; i++) //{ // for (int j = 0; j < detectResult[i].size(); j++)//left top right bottom score id // { // result[i].obj[j].id = 0; // result[i].obj[j].left = detectResult[i][j][0]; // result[i].obj[j].top = detectResult[i][j][1]; // result[i].obj[j].right = detectResult[i][j][2]; // result[i].obj[j].bottom = detectResult[i][j][3]; // result[i].obj[j].confidence = detectResult[i][j][4]; // result[i].obj[j].index = detectResult[i][j][5] - 1; // } // //} return SUCCESS; }