#include "VPTProcess.h" #ifndef _MSC_VER #include #endif #include #include #include "ObjCls.h" #include #include "ErrorInfo.h" #include "fstream" #include //#include "mobileshift0.6_mergeBNALL.h" //#include "ga_mobile_10_SSD10_416x224_416x224_mobile_shift2_mindim06_iter_46107_mergeBNALL.h" //#include "test_mergeBNALLGPU.h" //#include "ga_mobile_10cls_SSD_512x512_mobile_10clsv1_iter_200000_mergeBNALL.h" #include "../model/vptModeTrt/ga_vpt_init_net.h" #include "../model/vptModeTrt/ga_vpt_predict_net.h" #include "../model/vptModeTrt/ga_trt_fpn_vpt_calibrator.h" #include "../model/hidemodel_caffe_1108/ga_vpt_init_net_caffe2.h" #include "../model/hidemodel_caffe_1108/ga_vpt_predict_net_caffe2.h" #include "vpt_fpn_plugin_factory.h" //#include "vpt.h" #include "MutliSourceVideoProcess.h" //struct vpt_handle //{ // void* det_handle; // vector taskTrackers; //}; typedef struct objDetector { void* det_handle; float threshold; VPT_FPNPluginFactory tensorrt_plugin_factory; int licence_status = -1;//鎺堟潈鐩稿叧鍙傛暟 int thrd_status = -1; //鎺堟潈鐩稿叧鍙傛暟 boost::thread thrd; //鎺堟潈鐩稿叧鍙傛暟 vector taskTrackers; objDetector() { det_handle = NULL; threshold = 0.6; } }objDetector; //int VPT_Init(void *&handle, VPTProcess_PARAM param) //{ // int ret = SUCCESS; ////#ifdef AUTHORIZATION ////#ifdef _WIN32 //// if (SUCCESS == (ret = sy_licence(productSN))) ////#elif __linux__ //// char* wtime = new char[15]; //// memset(wtime, 0, 15); //// if (SUCCESS == (ret = sy_licence(productSN, &wtime))) ////#endif ////#else //// ret = sy_time_check(2021, 4); //// if (ret == SUCCESS) ////#endif //// { //// cuInit(0); //// int device_count = 0; //// cuDeviceGetCount(&device_count); //// //// if (param.gpuid >= device_count) //// { //// printf("\nGPU_ID PARAM WRONG!\n"); //// return GPUID_PARAM_ERROR; //// } // // objDetector * tools = new objDetector; // // ctools_init_params vpt_param; // vpt_param.thres_ = param.threshold; // vpt_param.log_level_ = 0; // vpt_param.device_type_ = param.mode; // vpt_param.device_id_ = param.gpuid; // vpt_param.engine_type_ = param.engine; // vpt_param.model_type_ = MODEL_FPN; // //#ifdef _MSC_VER // vpt_param.weight_file_ = NULL; // vpt_param.net_file_ = NULL; //#else // vpt_param.weight_file_ = NULL; // vpt_param.net_file_ = NULL; //#endif // // // vpt_param.data_process_str_ = param.preprocess_param; // vpt_param.need_im_info_ = 1; // true // // if (vpt_param.engine_type_ == ENGINE_MCAFFE2) // { // //caffe2妯″瀷棰勫鐞嗗弬鏁? // vpt_param.data_process_str_ = // "CopyData_CPU2GPU_U8;" // "TypeConvert_U8_F32;" // "ResizePad_F32_F32,test_size,720,test_max_size,1280,fpn_coarsest_stride,32," // "submean_b,103.94,submean_g,116.78,submean_r,123.68," // "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" // "NHWC2NCHW_F32" // ; // // vpt_param.weight_array_ = (unsigned char*)ga_vpt_init_net_caffe2; // vpt_param.weight_array_len_ = ga_vpt_init_net_len_caffe2; // vpt_param.net_array_ = (unsigned char*)ga_vpt_predict_net_caffe2; // vpt_param.net_array_len_ = ga_vpt_predict_net_len_caffe2; // } // else if (vpt_param.engine_type_ == ENGINE_TENSORRT) // { // vpt_param.weight_array_ = (uint8_t*)ga_vpt_init_net; // vpt_param.weight_array_len_ = ga_vpt_init_net_len; // vpt_param.net_array_ = (uint8_t*)ga_vpt_predict_net; // vpt_param.net_array_len_ = ga_vpt_predict_net_len; // // vpt_param.trt_serialize_file_ = param.serialize_file;// NULL;// "FPN_VPT_ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // // //trt鐗堟湰棰勫鐞嗗弬鏁? // param.preprocess_param = // "CopyData_CPU2GPU_U8;" // "TypeConvert_U8_F32;" // "ResizeMaxPad_F32_F32,test_size,720,test_max_size,1280,max_height,736,max_width,1280," // "submean_b,103.94,submean_g,116.78,submean_r,123.68," // "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" // "NHWC2NCHW_F32" // ; // // memset(vpt_param.tensorrt_param_str_, 0, sizeof(vpt_param.tensorrt_param_str_)); // // int batch_size = param.max_batch; // std::string g_data_mode = "FP32"; // bool g_is_create_calibrator = false; // // sprintf(vpt_param.tensorrt_param_str_, "max_batchsize %d," // "data_mode %s," // "is_create_calibrator %d," // "input_names data im_info," // "output_names cls_prob bbox_pred_final rois_count_each_img", // batch_size, g_data_mode.c_str(), g_is_create_calibrator); // // vpt_param.tensorrt_calibrator_file_ = NULL;// "trt_fpn_vpt_calibrator"; // vpt_param.tensorrt_calibrator_array_len_ = ga_trt_fpn_vpt_calibrator_len;// "trt_fpn_vpt_calibrator"; // vpt_param.tensorrt_calibrator_array_ = (unsigned char*)ga_trt_fpn_vpt_calibrator;// "trt_fpn_vpt_calibrator"; // // vpt_param.tensorrt_plugin_factory_ptr_ = &(tools->tensorrt_plugin_factory); // } // // tools->threshold = param.threshold; // // int flag = ctools_init(&tools->det_handle, &vpt_param); // // if (SUCCESS != flag) // { // printf("VPT_Init(ERROR): Init failed\n"); // handle = NULL; // } // else // handle = (void*)tools; // // return flag; //// } //// else //// { //// return AUTHOR_ERROR; //// } ////#ifdef AUTHORIZATION ////#ifdef __linux__ //// if (wtime) //// { //// delete[] wtime; //// wtime = NULL; //// } ////#endif ////#endif //} int VPT_Init(void *&handle, VPTProcess_PARAM vparam) { objDetector *tools = new objDetector; ctools_init_params param; param.thres_ = vparam.threshold; param.log_level_ = 0; param.device_type_ = vparam.mode; param.device_id_ = vparam.gpuid; param.engine_type_ = vparam.engine; param.model_type_ = MODEL_FPN; param.weight_array_ = (uint8_t*)ga_vpt_init_net; param.weight_array_len_ = ga_vpt_init_net_len; param.net_array_ = (uint8_t*)ga_vpt_predict_net; param.net_array_len_ = ga_vpt_predict_net_len; param.trt_serialize_file_ = vparam.serialize_file;// NULL;// "FPN_VPT_ABCDEFGHIJKLMNOPQRSTUVWXYZ"; //trt鐗堟湰棰勫鐞嗗弬鏁? memset(param.tensorrt_param_str_, 0, sizeof(param.tensorrt_param_str_)); int batch_size = vparam.max_batch; std::string g_data_mode = "INT8"; bool g_is_create_calibrator = false; sprintf(param.tensorrt_param_str_, "max_batchsize %d," "data_mode %s," "is_create_calibrator %d," "input_names data im_info," "output_names cls_prob bbox_pred_final rois_count_each_img", batch_size, g_data_mode.c_str(), g_is_create_calibrator); param.tensorrt_calibrator_file_ = NULL;// "trt_fpn_vpt_calibrator"; param.tensorrt_calibrator_array_len_ = ga_trt_fpn_vpt_calibrator_len;// "trt_fpn_vpt_calibrator"; param.tensorrt_calibrator_array_ = (unsigned char*)ga_trt_fpn_vpt_calibrator;// "trt_fpn_vpt_calibrator"; param.tensorrt_plugin_factory_ptr_ = &(tools->tensorrt_plugin_factory); //param.data_process_str_ = // "TypeConvert_U8_F32;" // "ResizePad_F32_F32,test_size,540,test_max_size,1280,fpn_coarsest_stride,32," // "submean_b,103.94,submean_g,116.78,submean_r,123.68," // "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" // "NHWC2NCHW_F32" // ; param.data_process_str_ = //"CopyData_CPU2GPU_U8;" "TypeConvert_U8_F32;" "ResizeMaxPad_F32_F32,test_size,720,test_max_size,1280,max_height,736,max_width,1280," "submean_b,103.94,submean_g,116.78,submean_r,123.68," "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" "NHWC2NCHW_F32" ; /* param.data_process_str_ = "CopyData_CPU2GPU_U8;" "TypeConvert_U8_F32;" "ResizePad_F32_F32,test_size,720,test_max_size,1280,fpn_coarsest_stride,32," "submean_b,103.94,submean_g,116.78,submean_r,123.68," "variance_rev_b,0.017,variance_rev_g,0.017,variance_rev_r,0.017;" "NHWC2NCHW_F32" ; */ int flag = ctools_init(&(tools->det_handle), ¶m); if (SUCCESS != flag) { if (tools) { delete tools; tools = NULL; } } else { handle = tools; } return flag; } void tmp_test(void * handle) { objDetector *tools = (objDetector*)handle; const int batchsize = 5; sy_img images[batchsize]; cv::Mat img[batchsize]; char strpath[1024]; memset(strpath, 0, sizeof(strpath)); for (int b = 0; b < batchsize; b++) { #ifdef _MSC_VER sprintf(strpath, "../../../data/%d.jpg", b);// b); #else sprintf(strpath, "0.jpg", b); #endif img[b] = cv::imread(strpath); images[b].set_data(img[b].cols, img[b].rows, img[b].channels(), img[b].data); } //vpt_batch(handle, images, batchsize, &result); int count = 1; while (count--) { printf("begin vpt_batch\n"); vpt_batch(tools->det_handle, images, batchsize, &result); } for (int b = 0; b < batchsize; b++) { std::cout << "b: " << b << ", count: " << result[b].obj_count_ << std::endl; } } int VPT_Process_GPU(void * handle, sy_img * batch_img, int startBatch, int batchsize, vector& result, vector>& deleteObjectID, vector>& unUsedResult) { tmp_test(handle); objDetector *tools = (objDetector*)handle; #ifndef _MSC_VER struct timeval first_time, second_time; gettimeofday(&first_time, NULL); #endif bool isUseDet = true; int channels = 3; #ifndef _MSC_VER if (0) { gettimeofday(&second_time, NULL); double time_val = (second_time.tv_sec - first_time.tv_sec) * 1000000 + second_time.tv_usec - first_time.tv_usec; printf("set data time_val(ms) = %lf\n", time_val / 1000.0); gettimeofday(&first_time, NULL); } #endif ctools_result *detresult; sy_img * batch_img_cpu = new sy_img[batchsize]; for (int i = 0; i < batchsize; ++i) { batch_img_cpu[i].data_ = (unsigned char *)malloc(batch_img[i].w_ * batch_img[i].h_ * batch_img[i].c_); cudaMemcpy(batch_img_cpu[i].data_, batch_img[i].data_, batch_img[i].w_ * batch_img[i].h_ * batch_img[i].c_, cudaMemcpyDeviceToHost); cv::Mat newVideoImg(batch_img[i].h_, batch_img[i].w_, CV_8UC3, batch_img_cpu[i].data_); char filename[260]; sprintf(filename, "pic/%d.jpg", i); cv::imwrite(filename, newVideoImg); free(batch_img_cpu[i].data_); } int res_status = ctools_process(tools->det_handle, batch_img, batchsize, &detresult); //for (int i = 0; i < batchsize; ++i) //{ // free(batch_img_cpu[i].data_); //} //delete[]batch_img_cpu; #ifndef _MSC_VER if (0) { gettimeofday(&second_time, NULL); double time_val = (second_time.tv_sec - first_time.tv_sec) * 1000000 + second_time.tv_usec - first_time.tv_usec; printf("process time_val(ms) = %lf\n", time_val / 1000.0); } #endif vector >> detectResult(batchsize); //杞寲涓鸿窡韪墍闇瑕佺殑杈撳叆 int batch_size = batchsize; #ifndef _MSC_VER if (0) { gettimeofday(&first_time, NULL); } #endif for (int b = 0; b < batch_size; b++) { ctools_result &cur_result = detresult[b]; printf("b=%d, count=%d\n", b, cur_result.obj_count_); for (int c = 0; c < cur_result.obj_count_ && c < MAX_OBJ_COUNT; c++) { float x1 = cur_result.obj_results_[c].data_[2]; float y1 = cur_result.obj_results_[c].data_[3]; float x2 = cur_result.obj_results_[c].data_[4]; float y2 = cur_result.obj_results_[c].data_[5]; float class_id = cur_result.obj_results_[c].data_[0]; float score = cur_result.obj_results_[c].data_[1]; int imgid = b; if (score >= THRESHOLD) { vector obj; obj.push_back(x1); obj.push_back(y1); obj.push_back(x2); obj.push_back(y2); obj.push_back(score); obj.push_back(class_id); detectResult[imgid].push_back(obj); } } } //exit(-1); #ifndef _MSC_VER if (0) { gettimeofday(&second_time, NULL); double time_val = (second_time.tv_sec - first_time.tv_sec) * 1000000 + second_time.tv_usec - first_time.tv_usec; printf("save result time_val(ms) = %lf\n", time_val / 1000.0); } #endif int resIndex = startBatch; int detectIndex = 0; #ifndef _MSC_VER if (0) { gettimeofday(&first_time, NULL); } #endif for (int i = startBatch; i < tools->taskTrackers.size(); i++) //batch?直????貌?同?母????? { printf("i=%d\n", i); if (!tools->taskTrackers[i].tracker.GetState()) { cout << "************************************ " << i << " pause" << endl; continue; } // cout << i << " " << taskTrackers[i].TaskID << " " << resIndex << endl; isUseDet = true; for (int j = 0; j < FusionInterval; j++) //??帧??????一帧?????? (FusionInterval - 1)帧 { int width = 1920; int height = 1080; //int objCount = 0; if (j == 0) { int objCount = tools->taskTrackers[i].tracker.update(/*tools->param.w*/width* tools->taskTrackers[i].ratioWidth, /*tools->param.h*/height* tools->taskTrackers[i].ratioHeight, isUseDet, detectResult[detectIndex], result[resIndex].obj, deleteObjectID[detectIndex]); result[resIndex].objCount = objCount; vector>().swap(detectResult[detectIndex]); detectResult[detectIndex].clear(); isUseDet = false; } else { VPT_Result unresult; VPT_ObjInfo obj[MAX_OBJ_COUNT]; unresult.objCount = tools->taskTrackers[i].tracker.update(/*tools->param.w*/width* tools->taskTrackers[i].ratioWidth, /*tools->param.h*/height* tools->taskTrackers[i].ratioHeight, isUseDet, detectResult[detectIndex], unresult.obj, deleteObjectID[detectIndex]); unUsedResult[resIndex].push_back(unresult); } } std::cout<<"tracked obj Count : "<>>().swap(detectResult); #ifndef _MSC_VER if (0) { gettimeofday(&second_time, NULL); double time_val = (second_time.tv_sec - first_time.tv_sec) * 1000000 + second_time.tv_usec - first_time.tv_usec; printf("tracking time_val(ms) = %lf\n", time_val / 1000.0); } #endif return SUCCESS; } void VPT_Release(void * handle) { objDetector *tools = (objDetector*)handle; if (tools) { if (tools->det_handle) { ctools_release(&tools->det_handle); tools->det_handle = NULL; } vector().swap(tools->taskTrackers); delete tools; tools = NULL; } } void AddTaskTracker(void * handle, const int taskID, const double rWidth, const double rHeight) { objDetector *tools = (objDetector*)handle; TaskTracker t; t.TaskID = taskID; t.ratioWidth = rWidth; t.ratioHeight = rHeight; tools->taskTrackers.push_back(t); } void FinishTaskTracker(void * handle, const int taskID) { objDetector *tools = (objDetector*)handle; for (int i = 0; i < tools->taskTrackers.size(); i++) { if (tools->taskTrackers[i].TaskID == taskID) { tools->taskTrackers.erase(tools->taskTrackers.begin() + i); break; } } } void PauseTaskTracker(void * handle, const int taskID) { objDetector *tools = (objDetector*)handle; for (int i = 0; i < tools->taskTrackers.size(); i++) { if (tools->taskTrackers[i].TaskID == taskID) { tools->taskTrackers[i].tracker.Pause(); break; } } } void RestartTaskTraker(void * handle, const int taskID) { objDetector *tools = (objDetector*)handle; for (int i = 0; i < tools->taskTrackers.size(); i++) { if (tools->taskTrackers[i].TaskID == taskID) { tools->taskTrackers[i].tracker.ReSet(); break; } } } void DrawTracker(void * handle, const int taskID, cv::Mat *img) { objDetector *tools = (objDetector*)handle; for (int i = 0; i < tools->taskTrackers.size(); i++) { if (tools->taskTrackers[i].TaskID == taskID) { tools->taskTrackers[i].tracker.addTracker(img); break; } } } int VPT_Process(void * handle, unsigned char ** bgr, int batchsize, VPT_Result *result) { // isUseDet = false; // } // } // resIndex++; // if (resIndex == batchsize) break; //} //vector >>().swap(detectResult); //vector>().swap(ssdResult); ///*detectResult.clear(); //ssdResult.clear();*/ return SUCCESS; } //#include void permute(float * image, int testWidth, int testHeight) { //cv::Mat host_image; float * host_image; //host_image.create(testHeight, testWidth, CV_32FC3); host_image = (float *)malloc(testHeight*testWidth * 3 * sizeof(float));; float *Host_img = new float[3 * testWidth * testHeight]{};//?????诖? float* image_data_original = image; //NPP_CHECK_CUDA(cudaMemcpy(Host_img, image_data_original, testWidth*testHeight * 3 * sizeof(float), cudaMemcpyDeviceToHost));//?????钥???????图???????? cudaMemcpy(Host_img, image_data_original, testWidth*testHeight * 3 * sizeof(float), cudaMemcpyDeviceToHost);//?????钥???????图???????? for (int j = 0; j < testHeight; j++) { float *pts = host_image + j * testWidth * 3; for (int i = 0; i < testWidth; i++) { //pts[3 * i] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 0]); //b //pts[3 * i + 1] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 1]); //g //pts[3 * i + 2] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 2]); //r pts[3 * i] = (Host_img[j * testWidth + i]); //b pts[3 * i + 1] = (Host_img[testWidth * testHeight + j * testWidth + i]); //g pts[3 * i + 2] = (Host_img[2 * testWidth * testHeight + j * testWidth + i]); //r } } cudaMemcpy(image_data_original, host_image, testWidth*testHeight * 3 * sizeof(float), cudaMemcpyHostToDevice); free(host_image); //cv::Mat showImg; //cv::resize(host_image, showImg, cv::Size(640, 480)); //cv::imshow("image", showImg); //cv::waitKey(0); } cv::Mat GpuMat2OpencvMat(unsigned char* image, int width, int height) { int testWidth = width; int testHeight = height; cv::Mat host_image; host_image.create(testHeight, testWidth, CV_8UC3); unsigned char *Host_img = new unsigned char[3 * testWidth * testHeight]{};//?????诖? unsigned char* image_data_original = image; cudaError_t code = cudaMemcpy(Host_img, image_data_original, testWidth*testHeight * 3 * sizeof(unsigned char), cudaMemcpyDeviceToHost);//?????钥???????图???????? if (code != 0) { printf("==========================================================error"); } std::ofstream outfile("decode.bin", ios::out | ios::binary); outfile.write((char*)Host_img, int(sizeof(char) * 1080 * 1920 * 3)); outfile.close(); cudaMemcpy(host_image.data, image_data_original, 1920 * testHeight * 3 * sizeof(unsigned char), cudaMemcpyDeviceToHost);//?????钥???????图???????? // for (int j = 0; j < host_image.rows; j++) // { // uchar *pts = host_image.ptr(j); // for (int i = 0; i < host_image.cols; i++) // { // //pts[3 * i] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 0]); //b // //pts[3 * i + 1] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 1]); //g // //pts[3 * i + 2] = cv::saturate_cast(Host_img[3 * (j*host_image.cols + i) + 2]); //r // pts[3 * i] = cv::saturate_cast(Host_img[j* host_image.cols*3 + 3 * i]); //b // pts[3 * i + 1] = cv::saturate_cast(Host_img[j* host_image.cols*3 + 3 * i + 1]); //g // pts[3 * i + 2] = cv::saturate_cast(Host_img[j* host_image.cols*3 + 3 * i + 2]); //r // } // } cv::imwrite("input3.jpg", host_image); return host_image; } int VPT_ProcessImage(void * handle, unsigned char ** bgr, int batchsize, VPT_Result * result) { //objDetector* tools = (objDetector*)handle; //if (bgr == NULL) //图?????荽??? // return IMG_DATA_ERROR; //vector imgs; ////int datasize = tools->param.w * tools->param.h * tools->param.c; //for (int i = 0; i < batchsize; i++) //{ // Mat img(tools->param.h, tools->param.w, tools->param.c, bgr[i]); // imgs.push_back(img); //} //vector> ssdResult; ////clock_t begin = clock(); //ssdResult = SSD_Detect(tools->detector, batchsize, imgs); // //vector >> detectResult(batchsize); //转??为????????要?????? //for (int i = 0; i < ssdResult.size(); ++i) //{ // const vector& d = ssdResult[i];// Detection format: [image_id, label, score, xmin, ymin, xmax, ymax) // const float score = d[2]; // int imgid = d[0]; // if (score >= THRESHOLD) //????为left top right bottom score id // { // vector obj; // obj.push_back(d[3] * imgs[d[0]].cols); // obj.push_back(d[4] * imgs[d[0]].rows); // obj.push_back(d[5] * imgs[d[0]].cols); // obj.push_back(d[6] * imgs[d[0]].rows); // obj.push_back(d[2]); // obj.push_back(d[1]); // detectResult[imgid].push_back(obj); // } //} //for (int i = 0; i < batchsize; i++) //{ // for (int j = 0; j < detectResult[i].size(); j++)//left top right bottom score id // { // result[i].obj[j].id = 0; // result[i].obj[j].left = detectResult[i][j][0]; // result[i].obj[j].top = detectResult[i][j][1]; // result[i].obj[j].right = detectResult[i][j][2]; // result[i].obj[j].bottom = detectResult[i][j][3]; // result[i].obj[j].confidence = detectResult[i][j][4]; // result[i].obj[j].index = detectResult[i][j][5] - 1; // } // //} return SUCCESS; }