#include #include "ErrorInfo.h" #include "VPT.h" #include "ObjCls.h" #include "utools.h" //model trt onnx 20210512 //#include "ga_vpt_onnx_net.h" //#include "ga_trt_yolo_vpt_calibrator.h" //model trt onnx 0715 //model pytorch_12cls_RenCheWu_yolov5m_hw640 // #include "ga_vpt_det_yolo_640x640.h" // #include "ga_trt_yolo_vpt_calibrator.h" // //model public pytorch_yolov5m_hw640 // #include "bky_vpt_det.h" //model public pytorch_yolov5m_h384_w640 #include "bky_vpt_det_384_640.h" //9 //老底层 模型 //#include "ga_vpt_predict_net.h" //#include "ga_vpt_init_net.h" //新trt的底层 跑caffe2的模型 //#include "ga_vpt_predict_net.h" //#include "ga_vpt_init_net.h" //13 //#include "ga_model.h" //#include "ga_model_init.h" //#define AUTHORIZATION int checkTime() { struct tm* info; int nYear, nMonth, nDay; time_t raw; time(&raw); info = localtime(&raw); nYear = info->tm_year + 1900; nMonth = info->tm_mon + 1; nDay = info->tm_mday; if (nYear == 2019 || (nYear == 2020 && nMonth < 4)) return 1; else { printf("版本过期，请联系开发商！中科院自动化所模式识别实验室图像视频组\n"); return -1; } } #include "authority.h" //#define productSN "92DAF36635F7492EB40D4C10DC67832F"//人车物SDK_linux_v4.0.4.190701_cuda10.0_cudnn7.5.0 #define productSN "7CF8B4797F9E441686145BED07F662DE"//人车物SDK_linux_v4.0.4.190701_cuda10.0_cudnn7.5.0 // map index_mp= {make_pair(0,0), make_pair(1,2), make_pair(2,4), make_pair(4,2), make_pair(5,5), make_pair(6,6), make_pair(7,8), make_pair(8,3), make_pair(10,1) , make_pair(11,2), make_pair(12,7)}; //12cls yolov5 map index_mp= {make_pair(0,0), make_pair(2,4), make_pair(5,4), make_pair(7,4)}; //coco yolov5 所有的车都映射成car void check_thread(objDetector * handle); int VPT_Init(void **handle, VPT_PARAM vparam, char* resDir, VIDEO_OBJECT_SNAPSHOT_CALLBACK VideoObjSnapshotCallBack/* = NULL*/) { //if (checkTime() != 1) // return FAILED; //int res = SUCCESS; int ret = SUCCESS; #ifdef AUTHORIZATION #ifdef _WIN32 if(SUCCESS == (ret = sy_licence(productSN))) #elif __linux__ char wtime[15]; memset(wtime, 0, 15); char * time = wtime; if (SUCCESS == (ret = sy_licence(productSN, &time))) #endif #endif { *handle = new objDetector; //申请对象 memcpy((void *)&((objDetector *)*handle)->param, (void *)&vparam, sizeof(VPT_PARAM)); int flag = 0; //返回值 默认为SUCCEEDED：0 //----------------- 13类结构化 ---------------------// /* param.initnetArray = (unsigned char*)ga_model_init; param.initnetArrayLen = ga_model_init_len; param.predictnetArray = (unsigned char*)ga_model; param.predictnetArrayLen = ga_model_len; param.mode = GPU_MODE; param.gpuid = vparam.gpuid; param.flag_glog = 1; param.test_size = 360; param.test_max_size = 640; param.fpn_coarsest_stride = 32; param.submean = "3 0 0 0"; param.variance = "3 255.0 255.0 255.0"; */ //----------------- 13类结构化 ---------------------// //----------------- 猫和狗结构化 ---------------------// /* param.initnetArray = (unsigned char*)ga_model_init; param.initnetArrayLen = ga_model_init_len; param.predictnetArray = (unsigned char*)ga_model; param.predictnetArrayLen = ga_model_len; param.mode = GPU_MODE; param.gpuid = 0; param.flag_glog = 1; param.test_size = 800; param.test_max_size = 1333; param.fpn_coarsest_stride = 32; param.submean = "3 102.9801 115.9465 122.7717"; param.variance = "3 1.0 1.0 1.0"; */ //----------------- 猫和狗结构化 ---------------------// //----------------- 9类结构化（自用单路人车物）---------------------// /* ctools_init_params vpt_param; vpt_param.log_level_ = 0; vpt_param.device_type_ = DEVICE_GPU; vpt_param.device_id_ = vparam.gpuid; vpt_param.engine_type_ = ENGINE_MCAFFE2; vpt_param.model_type_ = MODEL_FPN; #ifdef _MSC_VER //vpt_param.weight_file_ ="../../../model/vptModel/vpt_init_net"; //vpt_param.net_file_ = "../../../model/vptModel/vpt_predict_net"; #else vpt_param.weight_file_ = NULL; vpt_param.net_file_ = NULL; //vpt_param.weight_file_ = "vpt_model/init_net"; //vpt_param.net_file_ = "vpt_model/predict_net"; #endif vpt_param.weight_array_ = (unsigned char*)ga_vpt_init_net; vpt_param.weight_array_len_ = ga_vpt_init_net_len; vpt_param.net_array_ = (unsigned char*)ga_vpt_predict_net; vpt_param.net_array_len_ = ga_vpt_predict_net_len; vpt_param.need_im_info_ = 1; // true //"ResizePad_F32_F32,test_size,720,test_max_size,1280,fpn_coarsest_stride,32," //"NV12ToRGB_U8_U8,channel,3;" //"ResizePad_U8_F32,test_size,540,test_max_size,1280,fpn_coarsest_stride,32," //"NV12ToRGB_U8_F32,channel,3;" //"ResizePad_F32_F32,test_size,540,test_max_size,1280,fpn_coarsest_stride,32," vpt_param.data_process_str_ = "CopyData_CPU2GPU_U8;" "TypeConvert_U8_F32;" "ResizePad_F32_F32,test_size,720,test_max_size,1280,fpn_coarsest_stride,32," "submean_b,103.94,submean_g,116.78,submean_r,123.68," "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" "NHWC2NCHW_F32" ; ret = ctools_init(&(((objDetector *)handle)->detector), &vpt_param); */ /* ctools_init_params vpt_param; vpt_param.thres_ = 0.8; vpt_param.log_level_ = 0; vpt_param.device_type_ = DEVICE_GPU; vpt_param.device_id_ = vparam.gpuid; vpt_param.engine_type_ = ENGINE_MCAFFE2; vpt_param.model_type_ = MODEL_FPN; #ifdef _MSC_VER vpt_param.weight_file_ = NULL; vpt_param.net_file_ = NULL; #else vpt_param.weight_file_ = NULL; vpt_param.net_file_ = NULL; #endif vpt_param.data_process_str_ = "CopyData_CPU2GPU_U8;" "TypeConvert_U8_F32;" "ResizePad_F32_F32,test_size,720,test_max_size,1280,fpn_coarsest_stride,32," "submean_b,103.94,submean_g,116.78,submean_r,123.68," "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" "NHWC2NCHW_F32" ; vpt_param.need_im_info_ = 1; // true vpt_param.weight_array_ = (unsigned char*)ga_vpt_init_net; vpt_param.weight_array_len_ = ga_vpt_init_net_len; vpt_param.net_array_ = (unsigned char*)ga_vpt_predict_net; vpt_param.net_array_len_ = ga_vpt_predict_net_len; ret = ctools_init(&(((objDetector *)handle)->detector), &vpt_param); */ //----------------- 人车物检测新模型YOLO ---------------------// /* ctools_init_params vpt_param; vpt_param.thres_ = 0.5;//vparam.thres; vpt_param.log_level_ = 0; vpt_param.device_type_ =DEVICE_GPU; vpt_param.device_id_ = vparam.gpuid; vpt_param.engine_type_ = ENGINE_TENSORRT; vpt_param.data_process_str_ = "CopyData_CPU2GPU_U8;" "TypeConvert_U8_F32;" "ResizeMaxMidPad_F32_F32,test_size,640,test_max_size,640,max_height,640,max_width,640," "submean_b,0,submean_g,0,submean_r,0," "variance_rev_b,0.00392,variance_rev_g,0.00392,variance_rev_r,0.00392;" "BGR2RGB_F32_F32;" "NHWC2NCHW_F32" ; //param.preprocess_param; if (vpt_param.engine_type_ == ENGINE_MCAFFE2) { printf("ENGINE_MCAFFE2 not support!\n"); vpt_param.engine_type_ = ENGINE_TENSORRT; } if (vpt_param.engine_type_ == ENGINE_TENSORRT) { vpt_param.need_im_info_ = 0; // true vpt_param.model_type_ = MODEL_YOLOV5; vpt_param.net_array_ = (uint8_t*)ga_vpt_det_yolo_640x640; vpt_param.net_array_len_ = ga_vpt_det_yolo_640x640_len; vpt_param.trt_serialize_file_ = "VPT_DET";// NULL;// "FPN_VPT_ABCDEFGHIJKLMNOPQRSTUVWXYZ"; memset(vpt_param.tensorrt_param_str_, 0, sizeof(vpt_param.tensorrt_param_str_)); int batch_size = 20; std::string g_data_mode = "INT8"; bool g_is_create_calibrator = false; int g_is_onnx_model = 1; sprintf(vpt_param.tensorrt_param_str_, "max_batchsize %d," "data_mode %s," "is_create_calibrator %d," "is_onnx_model %d," "input_names images," "output_names output", batch_size, g_data_mode.c_str(), g_is_create_calibrator, g_is_onnx_model); //vpt_param.tensorrt_calibrator_file_ = "trt_yolo_vpt_calibrator"; vpt_param.tensorrt_calibrator_array_len_ = ga_trt_yolo_vpt_calibrator_len;// "trt_fpn_vpt_calibrator"; vpt_param.tensorrt_calibrator_array_ = (unsigned char*)ga_trt_yolo_vpt_calibrator;// "trt_fpn_vpt_calibrator"; } */ //pytorch_12cls_RenCheWu_yolov5m_hw640,模型类别有变化，process直接映射回之前的index ctools_init_params vpt_param; // vpt_param.thres_ = 0.5;//vparam.thres; vpt_param.thres_ = 0.45;//vparam.thres; vpt_param.log_level_ = 0; vpt_param.device_type_ =DEVICE_GPU; vpt_param.device_id_ = vparam.gpuid; vpt_param.engine_type_ = ENGINE_TENSORRT; // vpt_param.data_process_str_ = // "CopyData_CPU2GPU_U8;" // "TypeConvert_U8_F32;" // "ResizeMaxMidPad_F32_F32,test_size,640,test_max_size,640,max_height,640,max_width,640," // "submean_b,0,submean_g,0,submean_r,0," // "variance_rev_b,0.00392,variance_rev_g,0.00392,variance_rev_r,0.00392;" // "BGR2RGB_F32_F32;" // "NHWC2NCHW_F32" // ; vpt_param.data_process_str_ = "CopyData_CPU2GPU_U8;" "TypeConvert_U8_F32;" "ResizeMaxMidPad_F32_F32,test_size,384,test_max_size,640,max_height,384,max_width,640," "submean_b,0,submean_g,0,submean_r,0," "variance_rev_b,0.00392,variance_rev_g,0.00392,variance_rev_r,0.00392;" "BGR2RGB_F32_F32;" "NHWC2NCHW_F32" ; //param.preprocess_param; if (vpt_param.engine_type_ == ENGINE_MCAFFE2) { printf("ENGINE_MCAFFE2 not support!\n"); vpt_param.engine_type_ = ENGINE_TENSORRT; } if (vpt_param.engine_type_ == ENGINE_TENSORRT) { vpt_param.need_im_info_ = 0; // true vpt_param.model_type_ = MODEL_YOLOV5; // vpt_param.net_array_ = (uint8_t*)bky_vpt_det; // vpt_param.net_array_len_ = bky_vpt_det_len; vpt_param.net_array_ = (uint8_t*)bky_vpt_det_384_640; vpt_param.net_array_len_ = bky_vpt_det_384_640_len; char trt_serialize_file_[1024]; { if(vparam.serialize_file == nullptr) vparam.serialize_file = "."; string dir_path = vparam.serialize_file; int iLen = dir_path.length(); if (dir_path[iLen - 1] != '\\' && dir_path[iLen - 1] != '/') { dir_path += '/'; } // sprintf(trt_serialize_file_, "%sVPT_DET_MODEL20220211", dir_path.c_str()); sprintf(trt_serialize_file_, "%sVPT_DET_MODEL20220822", dir_path.c_str()); } printf("serialize_file path: %s\n",trt_serialize_file_ ); vpt_param.trt_serialize_file_ = trt_serialize_file_;//"VPT_DET"; memset(vpt_param.tensorrt_param_str_, 0, sizeof(vpt_param.tensorrt_param_str_)); int batch_size = 20; std::string g_data_mode = "FP32"; bool g_is_create_calibrator = false; int g_is_onnx_model = 1; sprintf(vpt_param.tensorrt_param_str_, "max_batchsize %d," "data_mode %s," "is_create_calibrator %d," "is_onnx_model %d," "input_names images," "output_names output", batch_size, g_data_mode.c_str(), g_is_create_calibrator, g_is_onnx_model); //vpt_param.tensorrt_calibrator_file_ = "trt_yolo_vpt_calibrator"; // vpt_param.tensorrt_calibrator_array_len_ = ga_trt_yolo_vpt_calibrator_len;// "trt_fpn_vpt_calibrator"; // vpt_param.tensorrt_calibrator_array_ = (unsigned char*)ga_trt_yolo_vpt_calibrator;// "trt_fpn_vpt_calibrator"; } ret = ctools_init(&(((objDetector *)*handle)->detector), &vpt_param); if(ret == SUCCESS) { /* cv::Mat img = cv::imread("test.jpg"); sy_img cur_img; cur_img.set_data(img.cols, img.rows, img.channels(), img.data); ctools_result *detresult; int count = 1000; vector< vector > detectResult; //转化为跟踪所需要的输入 while(count--) int res_status = ctools_process((((objDetector *)handle)->detector), &cur_img, 1, &detresult); cout << "finish Ctools_Detector" << endl; ctools_result &cur_result = detresult[0]; //(*result)[b].obj_count_ = cur_result.obj_count_ > MAX_DET_COUNT ? MAX_DET_COUNT : cur_result.obj_count_; for (int c = 0; c < cur_result.obj_count_ && c < 100; c++) { float score = cur_result.obj_results_[c].data_[1]; if (score >= THRESHOLD) //更改为left top right bottom score index { vector obj; obj.push_back(cur_result.obj_results_[c].data_[2]); obj.push_back(cur_result.obj_results_[c].data_[3]); obj.push_back(cur_result.obj_results_[c].data_[4]); obj.push_back(cur_result.obj_results_[c].data_[5]); obj.push_back(cur_result.obj_results_[c].data_[1]); obj.push_back(cur_result.obj_results_[c].data_[0] - 1); detectResult.push_back(obj); } } printf("============ finish det\n"); */ //----------------- 9类结构化（自用单路人车物）---------------------// //拌线初始化 ((objDetector *)*handle)->tracker = new Sort(/*lineArray, linecount, DETECTTYPE*/); ((objDetector *)*handle)->snapshotHelper = new SnapShot(resDir, VideoObjSnapshotCallBack, vparam.SnaoshotParameter); ((objDetector *)*handle)->frameCounter = FusionInterval; ((objDetector *)*handle)->isInitFrame = true; //----------------- 授权线程初始化 ---------------------// #ifdef AUTHORIZATION ((objDetector *)*handle)->licence_status = 0; ((objDetector *)*handle)->thrd = boost::thread(check_thread, ((objDetector *)*handle)); ((objDetector *)*handle)->thrd_status = 0; #endif } } /* } else { ret = AUTHOR_ERROR; } #ifdef __linux__ if(wtime) { delete[] wtime; wtime = NULL; } #endif */ return ret; } void VPT_ResetTracker(void * handle/*, int maxResultCountVPT_Line * lineArray = NULL, int linecount = 0*/) { objDetector* tools = (objDetector*)handle; /*if (tools->ssdResult) { delete[] tools->ssdResult; } tools->param.maxResultCount = maxResultCount; tools->ssdResult = new CDTOOLS_Result[maxResultCount];*/ tools->tracker->Release(); delete tools->tracker; tools->tracker = NULL; tools->tracker = new Sort(/*lineArray, linecount, DETECTTYPE*/); //新接口 无需流量统计和快照 } //辅助函数：数据预处理 float VPTmeanSub[3] = { 103.52, 116.28, 123.675 }; float VPTVariance[3] = { 57.375, 57.12, 58.395 }; void datapreprocess(float * blob, int width, int height, int channels, int batchsize) { int datasize_ = width * height; for (int c = 0; c < 3; c++) { for (int i = 0; i < width * height; i++) { blob[datasize_ * c + i] = ((blob[datasize_ * c + i]) - VPTmeanSub[c]) / VPTVariance[c]; } } } int VPT_ProcessLastFrame(void * handle/*, unsigned char * rgb, int width, int height, FT_Result * result*/) { objDetector* tools = (objDetector*)handle; //CDTOOLS_Result ssdResult[MAX_OBJ_COUNT_TEST]; vector< vector > detectResult; //转化为跟踪所需要的输入 int index = 0; bool isUseDet = false; vector deleteTrackers; int ObjCount = tools->tracker->updateLastFrame(deleteTrackers); //void SaveSnapshot(vector deleteTrackers); tools->snapshotHelper->SaveSnapshot(deleteTrackers); sleep(10); //等待一会，以便存完 220824 byzsh vector>().swap(detectResult); detectResult.clear(); vector().swap(deleteTrackers); deleteTrackers.clear(); return 0; } //int VPT_Process(void * handle, unsigned char *bgr, int width, int height, int channels, int frameCount, VPT_Result *result, vector &deleteTrackers, VPT_TrafficResult* traffic, bool showTrack) int VPT_Process(void * handle, unsigned char *bgr, int width, int height, int channels, int frameCount, VPT_Result *result,/* int maxResultCount,*/ bool showTrack) { objDetector* tools = (objDetector*)handle; //CDTOOLS_Result ssdResult[MAX_OBJ_COUNT_TEST]; #ifdef AUTHORIZATION if (tools->licence_status > -3) #endif { Mat img(height, width, CV_8UC3, bgr); cv::Mat tmp_img; img.copyTo(tmp_img); vector< vector > detectResult; //转化为跟踪所需要的输入 int index = 0; bool isUseDet = false; vector deleteTrackers; if (tools->frameCounter == FusionInterval || tools->isInitFrame) { // cout << "begin Ctools_Detector" << endl; sy_img cur_img; cur_img.set_data(img.cols, img.rows, img.channels(), img.data); ctools_result *detresult; int res_status = ctools_process(tools->detector, &cur_img, 1, &detresult); //cout << "finish Ctools_Detector" << endl; ctools_result &cur_result = detresult[0]; //(*result)[b].obj_count_ = cur_result.obj_count_ > MAX_DET_COUNT ? MAX_DET_COUNT : cur_result.obj_count_; for (int c = 0; c < cur_result.obj_count_ && c < tools->param.maxResultCount; c++) { float score = cur_result.obj_results_[c].data_[1]; int old_index = cur_result.obj_results_[c].data_[0]; // if (score >= THRESHOLD && (old_index!=3 && old_index!=9)/*index 为pytorch_12cls_RenCheWu_yolov5m_hw640模型添加index映射的操作*/) //更改为left top right bottom score index(未-1的原始数据 sort内部进行-1操作) if (score >= THRESHOLD && (old_index==0 || old_index==2 || old_index==5 || old_index==7)) // 只保留0-person 2-car 5-bus 7-truck { vector obj; obj.push_back(cur_result.obj_results_[c].data_[2]); obj.push_back(cur_result.obj_results_[c].data_[3]); obj.push_back(cur_result.obj_results_[c].data_[4]); obj.push_back(cur_result.obj_results_[c].data_[5]); obj.push_back(cur_result.obj_results_[c].data_[1]); //obj.push_back(cur_result.obj_results_[c].data_[0] + 1); //针对YOLO模型 内部-1 外部得+1 int new_index = index_mp[old_index] + 1; obj.push_back(new_index); detectResult.push_back(obj); } } //dResult->objCount = index; result->objCount = detectResult.size(); //VPT_Result temp; tools->isInitFrame = false; tools->frameCounter = 0; isUseDet = true; } tools->frameCounter++; //for (int j = 0; j < FusionInterval; j++) //跳帧：检测一帧，跟踪 (FusionInterval - 1)帧 //{ int objCount = tools->tracker->update(width, height, isUseDet, detectResult, result->obj, deleteTrackers); result->objCount = objCount; tools->snapshotHelper->SnapshotProcess(result, tmp_img, deleteTrackers, frameCount); vector>().swap(detectResult); detectResult.clear(); vector().swap(deleteTrackers); deleteTrackers.clear(); //} //} if (showTrack) tools->tracker->addTracker(&tmp_img); return 0; } /* else { return AUTHOR_ERROR; } */ } //int VPT_ProcessImage(void * handle, unsigned char * bgr, int width, int height, int channels, VPT_Result * result) //{ // objDetector* tools = (objDetector*)handle; // // if (bgr == NULL || width == 0 || height == 0) //图像数据错误 // return -1; // // clock_t begin = clock(); // // CDTOOLS_Result ssdResult[50]; // // int count = Ctools_Detector(tools->detector, bgr, width, height, 3, ssdResult, datapreprocess); // // clock_t end = clock(); // // for (int i = 0; i < count; i++) // { // const CDTOOLS_Result& d = ssdResult[i];// Detection format: [image_id, label, score, xmin, ymin, xmax, ymax) // const float score = d.boxinfo[5]; // // result->obj[i].id = i; // result->obj[i].left = d.boxinfo[0]; // bbout[i][0]; // result->obj[i].top = d.boxinfo[1]; //bbout[i][1]; // result->obj[i].right = d.boxinfo[2]; //bbout[i][2]; // result->obj[i].bottom = d.boxinfo[3]; //bbout[i][3]; // result->obj[i].confidence = d.boxinfo[4]; // result->obj[i].index = d.boxinfo[5] - 1; // bbout[i][4]; // } // // result->objCount = count; // return count; //} void VPT_Release(void **handle) { objDetector* tools = (objDetector*)*handle; VPT_ProcessLastFrame(tools); if (tools && tools->detector) { ctools_release(&tools->detector); tools->detector = NULL; } #ifdef AUTHORIZATION if(tools->thrd_status == 0) { tools->thrd.interrupt(); tools->thrd.join(); tools->thrd_status = -1; } #endif if (tools) { delete tools; tools = NULL; } } void check_thread(objDetector * handle) { int res = -1; #ifdef __linux__ char wtime[15]; memset(wtime, 0, 15); char * time = wtime; #endif while (1) { //printf("xxx check status on process...\n"); #ifdef _WIN32 res = sy_licence(productSN); #elif __linux__ res = sy_licence(productSN, &time); //printf("--------------wtime in thread: %s, status: %d\n", wtime, licence_status); #endif if (res < 0) { handle->licence_status= handle->licence_status - 1; } else { if (handle->licence_status < 0) { handle->licence_status = 0; } } boost::this_thread::sleep(boost::posix_time::seconds(300)); //5min //boost::this_thread::sleep(boost::posix_time::milliseconds(10); } }