#include "MultiSourceVideoProcess.h" #ifdef _MSC_VER #include #include //#include "putText.h" #endif #include #include #include #include #ifdef POST_USE_RABBITMQ #include "helpers/gen_json.hpp" #endif #include "helpers/logger.hpp" #include "helpers/os_helper.hpp" //#define AUTHORIZATION //#define DQ_AUTHORIZATION #ifdef DQ_AUTHORIZATION #include "license_validator.h" #ifdef _MSC_VER #define productSN "2AC69C4638EF46c884BD2BF132FF41D9" // 大千定制-定制授权ID #else #define productSN "683E9D5E56474da5A4C2D3EA9A00568E" // 大千定制-定制授权ID #endif #endif #include "authority.h" #ifdef AUTHORIZATION #include #ifdef _MSC_VER #define productSN "4ACFCBE67EF645AB8F0B4CFCDD1926F1" // WINDOWS #else #define productSN "4FD45501D5104F0C8C4BE530FC872F46" // LINUX #endif #endif //#define LOG_INFO // #define SKIP_FRAME 5 // #define WAIT_TIME 1 // 等待时间 分钟 -byzsh 220614 void check_thread(void *handle); DWORD ThreadProcess(LPVOID param); CMultiSourceVideoProcess::CMultiSourceVideoProcess() = default; CMultiSourceVideoProcess::~CMultiSourceVideoProcess() = default; /* @FinishProcessThread * @Description: 分析结束 最终释放全局资源+释放算法句柄资源 */ int CMultiSourceVideoProcess::FinishProcessThread() { if (thread_status_ == 0) { thread_.interrupt(); thread_.join(); thread_status_ = -1; } ProcessThread.interrupt(); // interrupt thread ProcessThread.join(); // waiting thread finish VPT_Release(VPT_Handle_); ((save_snapshot_reprocessing *)m_save_snapshot_reprocessing)->save_snapshot_reprocessing_release(); if (m_save_snapshot_reprocessing) { delete m_save_snapshot_reprocessing; m_save_snapshot_reprocessing = nullptr; } m_task_param_manager->task_param_manager_release(); DxDecoderInterface::UninitDecoderModule(); return 1; } /* @InitAlgorthim * @Description: 初始化全局参数 + 根据配置初始化算法模型 */ int CMultiSourceVideoProcess::InitAlgorthim(tsl_aiplatform_param vptParam) { licence_status_ = -1; thread_status_ = -1; int ret = SUCCESS; #ifdef AUTHORIZATION #ifdef _WIN32 if (SUCCESS == (ret = sy_licence(productSN))) #elif __linux__ char wtime[15]; memset(wtime, 0, 15); char *time = wtime; if (SUCCESS == (ret = sy_licence(productSN, &time))) #endif // #else // ifdef AUTHORIZATION ret = sy_time_check(2023, 2); // license_check(param.auth_license, productSN);// if (ret == SUCCESS) #endif // ifdef AUTHORIZATION { /*初始化全局参数*/ viewTaskID = -1; TaskInPlay = 0; TotalTask = 0; ProcessFlag = false; SourceFlag = false; mModeSnapshotVideo = "cpu"; mModeSnapshotLittle = "cpu"; taskFinishCallbackFunc = nullptr; taskObjInfoCallbackFunc = nullptr; set_default_logger(LogLevel(vptParam.log_level), "multi_source_video_process", vptParam.log_path, vptParam.log_mem, vptParam.log_mem); cuInit(0); int device_count = 0; cuDeviceGetCount(&device_count); if (vptParam.gpuid >= device_count) { LOG_ERROR("gpu id ge host gpu device num ({} vs {})", vptParam.gpuid, device_count); return GPUID_PARAM_ERROR; } CUdevice dev = 0; size_t memSize = 0; dev = vptParam.gpuid; CUresult rlt = CUDA_SUCCESS; rlt = cuDeviceTotalMem(&memSize, dev); gpu_total_memory = (float)memSize / (1024 * 1024); if (gpu_total_memory < 9000) // small gpu memory section_batch_size_ = 10; else section_batch_size_ = 20; /*初始化各个模块：任务管理、快照处理、快照保存、MQ*/ m_task_param_manager = task_param_manager::getInstance(); m_snapshot_reprocessing = snapshot_reprocessing::getInstance(); m_save_snapshot_reprocessing = new save_snapshot_reprocessing(); #ifdef POST_USE_RABBITMQ mq_manager_ = new mq::Manager(); #endif /*初始化各个算法模块*/ /* Init Model */ { VPTProcess_PARAM param{}; { param.gpuid = vptParam.gpuid; // param.threshold = 0.6; param.threshold = 0.45; param.max_batch = section_batch_size_; param.serialize_file = "./serialize_file/FPN_VPT"; param.auth_license = /*vptParam.auth_license*/ "sy_tsl_aiplatform_sdk_2021"; gpu_id_ = vptParam.gpuid; } VPT_Handle_ = nullptr; if (0 != (ret = VPT_Init(VPT_Handle_, param))) { LOG_FATAL("Init VPT failed"); return ret; } #ifdef WITH_SECOND_PROCESS if (!takeaway_member_.init(vptParam.gpuid)) { LOG_FATAL("Init TakeAwayMember failed"); return FALSE; } if (!fight_fall_cls_.init(vptParam.gpuid)) { LOG_FATAL("Init FightFellCls failed"); return FALSE; } if (m_human_gather_statistics.human_gather_statistics_init() != 0) { LOG_FATAL("Init Human Gather failed"); return FALSE; } { hat_callsmoke_vestuniform_det_param init_param; { init_param.gpuid = vptParam.gpuid; init_param.mode = init_param.gpuid >= 0 ? DEVICE_GPU : DEVICE_CPU; init_param.engine = ENGINE_TENSORRT; init_param.max_batch = section_batch_size_; init_param.auth_license = "shiyu_zhian_2021_jetson_sdk"; init_param.serialize_file = "./serialize_file/pedestrian_safety_det"; } if (!pedestrian_safety_detector_.init(init_param)) { LOG_FATAL("Init Pedestrian Safety failure"); return FALSE; } } { // pedestrian_vehicle_retrograde_.init() } #endif #ifdef WITH_FACE_DET_SS // 人脸检测初始化 facedet_ai_engine_param fd_param; fd_param.sdk_param.gpuid = vptParam.gpuid; fd_param.sdk_param.mode = DEVICE_GPU; fd_param.sdk_param.log = SY_CONFIG_CLOSE; fd_param.sdk_param.thresld = 0.6; fd_param.sdk_param.facial_fea_point_config = SY_CONFIG_OPEN; fd_param.sdk_param.pose_config = SY_CONFIG_OPEN; fd_param.sdk_param.quality_config = SY_CONFIG_CLOSE; fd_param.sdk_param.score_config = SY_CONFIG_OPEN; fd_param.sdk_param.max_result_count = 50; fd_param.sdk_param.max_batch_size_detect = section_batch_size_; fd_param.sdk_param.max_batch_size_ldmk = section_batch_size_; fd_param.sdk_param.max_batch_size_score = section_batch_size_; fd_param.sdk_param.max_batch_size_pose = section_batch_size_; fd_param.sdk_param.serialize_file = "./serialize_file/FD"; fd_param.sdk_param.auth_license = "sy_tsl_aiplatform_sdk_2021"; if (0 > m_face_det_ai_engine.init_ai_engine(fd_param)) { LOG_FATAL("Init face detection failed"); return FALSE; } #endif } if (ret == SUCCESS) // { licence_status_ = 0; #ifdef AUTHORIZATION thread_ = boost::thread(check_thread, this); #endif thread_status_ = 0; } } else { return AUTHOR_ERROR; } #ifdef AUTHORIZATION #ifdef __linux__ if (wtime) { delete[] wtime; wtime = NULL; } #endif // #ifdef __linux__ #endif // #ifdef AUTHORIZATION return ret; } DWORD ThreadProcess(LPVOID param) { CMultiSourceVideoProcess *pThreadParam = (CMultiSourceVideoProcess *)param; auto task_has_vpt_algor = [&pThreadParam](const std::string &task_id) { //! TODO: create enum iterator. auto algor_map = pThreadParam->m_task_param_manager->get_task_other_param(task_id); if (algor_map == nullptr) return false; return (algor_map->find(algorithm_type_t::HUMAN_GATHER) != algor_map->end() || algor_map->find(algorithm_type_t::HUMAN_SNAPSHOT) != algor_map->end() || algor_map->find(algorithm_type_t::NONMOTOR_VEHICLE_SNAPSHOT) != algor_map->end() || algor_map->find(algorithm_type_t::SMOKING_DET) != algor_map->end() || algor_map->find(algorithm_type_t::NO_REFLECTIVE_CLOTHING) != algor_map->end() || algor_map->find(algorithm_type_t::NO_SAFETY_HELMET) != algor_map->end() || algor_map->find(algorithm_type_t::CALL_PHONE_DET) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_SNAPSHOT) != algor_map->end() || algor_map->find(algorithm_type_t::TAKEAWAY_MEMBER_CLASSIFICATION) != algor_map->end() || algor_map->find(algorithm_type_t::PEDESTRIAN_FALL) != algor_map->end() || algor_map->find(algorithm_type_t::PEDESTRIAN_FIGHT) != algor_map->end() || algor_map->find(algorithm_type_t::PEDESTRIAN_RETROGRADE) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_RETROGRADE) != algor_map->end() || algor_map->find(algorithm_type_t::PEDESTRIAN_TRESPASS) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_TRESPASS) != algor_map->end()); }; auto task_has_face_algor = [&pThreadParam](const std::string &task_id) { auto algor_map = pThreadParam->m_task_param_manager->get_task_other_param(task_id); return algor_map->find(algorithm_type_t::FACE_SNAPSHOT) != algor_map->end(); }; cudaError_t cudaStatus = {}; sy_img *batch_img = new sy_img[20]{}; DxGPUFrame frame = {}; #ifdef _MSC_VER LARGE_INTEGER nFreq, nBeginTime, nEndTime; LARGE_INTEGER nSaveBeginTime, nSaveEndTime; QueryPerformanceFrequency(&nFreq); #endif cudaSetDevice(pThreadParam->gpu_id_); cuda_common::setColorSpace2(ITU709, 0); CHECK(cudaMalloc((void **)&pThreadParam->FrameTemp, 3 * 4096 * 4096 * sizeof(unsigned char))); long long costTime1 = 0; int total_count = 0; auto begintime1 = std::chrono::system_clock::now(); set k; #ifdef MTASK_DEBUG_ #endif map task_id_to_n_frame; static int test_frame = 0; /* 主流程 while循环，循环内容< 任务处理 -> 解码数据获取 -> 一级检测器 -> 快照更新 -> 每帧后续分析 -> * delete目标的最后帧分析 >*/ while (true) { CHECK(cudaGetLastError()); /* step1. 授权check */ if (pThreadParam->licence_status_ <= -3) { LOG_FATAL("authority failed!"); break; } /* step2. 处理缓存中待更新的任务 */ { std::lock_guard l(pThreadParam->taskMutex); pThreadParam->OperatorTask(); } pThreadParam->taskCondVar.notify_all(); /* step3. 获取当前任务的情况，任务都结束则退出主循环，没有PLAY任务循环等待 */ int curTaskSize = pThreadParam->system_all_tasks_.size(); // 总任务数 int task_play = 0; // 当前活跃任务数 int task_finish = 0; // 当前已结束任务数 for (auto cur_task : pThreadParam->system_all_tasks_) { if (cur_task.second.task_state == PLAY) ++task_play; if (cur_task.second.task_state == FINISH) ++task_finish; } if (task_finish >= curTaskSize) // have no decode video, break { std::lock_guard l(pThreadParam->taskMutex); if (pThreadParam->HasNewTask()) continue; else break; } if (task_play <= 0) { Sleep(30); continue; } k.clear(); pThreadParam->TaskInPlayID.clear(); // /* step4. 获取各路解码数据 */ /* get one frame from per task. */ auto getdatat1 = std::chrono::system_clock::now(); //----zsh 220614 添加超时等待 getdata_flag: for (auto &cur_task : pThreadParam->system_all_tasks_) { task_resource &cur_task_resource = cur_task.second; /* 当前路数处于PLAY状态 获取解码数据 */ if (k.find(cur_task.first) == k.end() && cur_task_resource.task_state == PLAY) { if (cur_task_resource.taskcuvid->DxDecoderIsRun()) { /* 解码器完成解码 解码数据存在 */ if (cur_task_resource.taskcuvid->DxLockFrame(&frame) == 0) { /* 第一次获取 申请显存 */ if (!cur_task_resource.task_algorithm_data.frame && frame.width > 0 && frame.height > 0) { cudaError_t cudaStatus = cudaMalloc((void **)&cur_task_resource.task_algorithm_data.frame, 3 * frame.size * frame.height * sizeof(float)); if (cudaStatus != cudaSuccess) { LOG_ERROR("here cudaMalloc m_pRGBData[0] failed! error: {}", cudaGetErrorString(cudaStatus)); break; } cur_task_resource.task_algorithm_data.height = frame.height; cur_task_resource.task_algorithm_data.width = frame.width; cur_task_resource.task_algorithm_data.size = frame.size; if (task_has_vpt_algor(cur_task.first)) AddTaskTracker(pThreadParam->VPT_Handle_, cur_task.first); #ifdef WITH_FACE_DET_SS if (task_has_face_algor(cur_task.first)) pThreadParam->m_face_det_ai_engine.operator_tracker(cur_task.first, ADDTASK, SKIP_FRAME); // 跳帧数暂时写死 #endif } // 可省略2次cpy, zqli /* 已申请好显存 直接cpy */ // copy decoded data and convert to rgb color space. if (cur_task_resource.task_algorithm_data.frame) { cudaStatus = cudaMemcpy(cur_task_resource.task_algorithm_data.frame, frame.frame, 1.5 * frame.size * frame.height * sizeof(unsigned char), cudaMemcpyDeviceToDevice); if (cudaStatus != cudaSuccess) LOG_ERROR("cudaMemcpy failed: {}", cudaGetErrorString(cudaStatus)); cudaStatus = cuda_common::NV12ToRGBnot( (CUdeviceptr)cur_task_resource.task_algorithm_data.frame, cur_task_resource.task_algorithm_data.size, (unsigned char *)pThreadParam->FrameTemp, cur_task_resource.task_algorithm_data.width, cur_task_resource.task_algorithm_data.height); if (cudaStatus != cudaSuccess) { cur_task_resource.taskcuvid->DxUnlockFrame(); CHECK(cudaDeviceSynchronize()); continue; } CHECK(cudaDeviceSynchronize()); cudaStatus = cudaMemcpy(cur_task_resource.task_algorithm_data.frame, pThreadParam->FrameTemp, 3 * cur_task_resource.task_algorithm_data.width * cur_task_resource.task_algorithm_data.height * sizeof(unsigned char), cudaMemcpyDeviceToDevice); if (cudaStatus != cudaSuccess) { cur_task_resource.taskcuvid->DxUnlockFrame(); CHECK(cudaDeviceSynchronize()); continue; } auto &task_id = cur_task.first; k.insert(task_id); ++task_id_to_n_frame[task_id]; pThreadParam->TaskInPlayID.insert(task_id); } else { LOG_ERROR("NOT MALLOC: pThreadParam->tasks[i].taskDataToBackup.frame {}", cur_task_resource.task_algorithm_data.frame); } cur_task_resource.taskcuvid->DxUnlockFrame(); } } else /* 解码出现异常 DECODEERROR，结束该路任务 */ { cur_task_resource.task_state = DECODEERROR; task_play--; if (!FinishTaskTracker(pThreadParam->VPT_Handle_, cur_task.first)) LOG_ERROR("Finish VPT Tracker failed, task_id: {}", cur_task.first); } } } // continue; if (task_play <= 0) { Sleep(30); continue; } /* 解码数据未凑满 则循环等待(可加超时等待) */ //------------------------------------------------------------------------------------------------- // if (k.size() < task_play) { // boost::this_thread::sleep(boost::posix_time::milliseconds(1)); // goto getdata_flag; // } //----zsh 220614 添加超时等待 auto getdatat2 = std::chrono::system_clock::now(); auto getdata_time = std::chrono::duration_cast(getdatat2 - getdatat1).count(); // auto getdata_time = std::chrono::duration_cast(getdatat2 - getdatat1).count(); if (k.size() < task_play && getdata_time < WAIT_TIME) { boost::this_thread::sleep(boost::posix_time::milliseconds(1)); goto getdata_flag; } for (auto &cur_task : pThreadParam->system_all_tasks_) { auto &task_id = cur_task.first; task_resource &cur_task_resource = cur_task.second; if (k.find(cur_task.first) == k.end() && cur_task_resource.task_state == PLAY) { cout << "------------decode too slow: " << cur_task.first << endl; // 解码慢就结束任务？ LOG_INFO("decode too slow: task id: {} task state: {}", cur_task.first.c_str(), cur_task.second.task_state); pThreadParam->FinishTask(cur_task.first, false); pThreadParam->FinishDecode(cur_task.first); task_play--; } } //------------------------------------------------------------------------------------------------- CHECK(cudaDeviceSynchronize()); /* step5. 凑齐的解码数据 拼batch */ int idx = 0; sy_img *batch_img = new sy_img[task_play]; for (auto iter : pThreadParam->TaskInPlayID) // 先处理第一个MAX_BATCH { task_resource &cur_task_resource = pThreadParam->system_all_tasks_[iter]; batch_img[idx++].set_data(cur_task_resource.task_algorithm_data.width, cur_task_resource.task_algorithm_data.height, 3, (unsigned char *)cur_task_resource.task_algorithm_data.frame); } /* step6. 开始一级检测器 */ /* for vpt detection. */ { decltype(pThreadParam->TaskInPlayID) vpt_interest_task_id; std::vector vpt_interest_imgs(0); /* 确定待检测的任务和图片 */ int _idx = 0; for (auto _task_id_iter = pThreadParam->TaskInPlayID.begin(); _task_id_iter != pThreadParam->TaskInPlayID.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; if (!task_has_vpt_algor(task_id)) continue; vpt_interest_task_id.insert(task_id); vpt_interest_imgs.emplace_back(batch_img[_idx]); } // vpt_interest_imgs.reverse(vpt_interest_imgs.size()); /* 待检测的图片不为空 开始检测 */ if (!vpt_interest_imgs.empty()) { vector> deleteObjectID; deleteObjectID.resize(vpt_interest_task_id.size()); vector> unUsedResult; vector vptResult(0); /* 一级检测器，内部已完成跟踪操作 */ VPT_Process_GPU(pThreadParam->VPT_Handle_, vpt_interest_imgs.data(), vpt_interest_imgs.size(), vptResult, deleteObjectID, unUsedResult); // do det & track. /* 每帧都需要进行的操作函数（如人数统计功能） */ // pThreadParam->everyframe_process(vpt_interest_task_id, batch_img, vptResult); pThreadParam->everyframe_process(vpt_interest_task_id, vpt_interest_imgs.data(), vptResult); // modified by zsh /* 根据任务的配置 对目标进行筛选 */ pThreadParam->m_snapshot_reprocessing->screen_effective_snapshot(vpt_interest_task_id, vptResult); // #ifdef MTASK_DEBUG_ int idx_tmp = 0; for (auto task_id : vpt_interest_task_id) LOG_DEBUG("after screen_effective_snapshot task_id {} obj count {}", task_id, vptResult[idx_tmp++].obj_count); #endif /* 快照优选（内部可实现不同的快照优选策略） */ pThreadParam->m_snapshot_reprocessing->update_bestsnapshot(vpt_interest_task_id, vpt_interest_imgs.data(), vptResult, deleteObjectID); #ifdef WITH_SECOND_PROCESS /* for pedestrian safety det. 行人安全分析算法模块 */ { decltype(pThreadParam->TaskInPlayID) interest_task_id; decltype(vptResult) interest_vpt_result; vector interest_imgs(0); int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = pThreadParam->m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::NO_REFLECTIVE_CLOTHING) != algor_map->end() || algor_map->find(algorithm_type_t::NO_SAFETY_HELMET) != algor_map->end() || algor_map->find(algorithm_type_t::CALL_PHONE_DET) != algor_map->end() || algor_map->find(algorithm_type_t::SMOKING_DET) != algor_map->end()) { interest_task_id.insert(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } if (!interest_imgs.empty()) pThreadParam->pedestrian_safety_detector_.update_mstreams(interest_task_id, interest_imgs.data(), interest_vpt_result); } /* for retrograde & trespass algor 逆行&非法闯入算法模块 */ { decltype(pThreadParam->TaskInPlayID) interest_task_id; decltype(vptResult) interest_vpt_result; vector interest_imgs(0); decltype(pThreadParam->TaskInPlayID) trespass_interest_task_id; decltype(vptResult) trespass_interest_vpt_result; decltype(deleteObjectID) trespass_interest_deleteobjs; vector trespass_interest_imgs(0); int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = pThreadParam->m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::PEDESTRIAN_RETROGRADE) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_RETROGRADE) != algor_map->end()) { interest_task_id.insert(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } if (algor_map->find(algorithm_type_t::PEDESTRIAN_TRESPASS) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_TRESPASS) != algor_map->end()) { trespass_interest_task_id.insert(task_id); trespass_interest_imgs.emplace_back(vpt_interest_imgs[_idx]); trespass_interest_vpt_result.emplace_back(vptResult[_idx]); // printf("push back deleteObjectID\n"); trespass_interest_deleteobjs.emplace_back(deleteObjectID[_idx]); // printf("end push back deleteObjectID\n"); } } if (!interest_imgs.empty()) pThreadParam->pedestrian_vehicle_retrograde_.update_mstreams(interest_task_id, interest_imgs.data(), interest_vpt_result); if (!trespass_interest_imgs.empty()) { pThreadParam->pedestrian_vehicle_trespass_.update_mstreams( trespass_interest_task_id, trespass_interest_imgs.data(), trespass_interest_vpt_result, trespass_interest_deleteobjs); } } #endif /* for snapshot algorithm. 轨迹结束目标 做最后的结果返回（当前返回算法结果+快照保存路径）*/ { auto task_iter = vpt_interest_task_id.begin(); for (int i = 0; i < deleteObjectID.size(); i++, ++task_iter) // loop taskId. { for (int &j : deleteObjectID[i]) // loop algor type. { OBJ_KEY obj_key = {*task_iter, j}; pThreadParam->endframe_obj_process(obj_key, algorithm_type_t::PLACEHOLDER); } vector().swap(deleteObjectID[i]); // free. } vector>().swap(deleteObjectID); // free. } #ifdef WITH_SECOND_PROCESS /* for fight fall cls. sync code. 打架跌倒算法模块 */ { auto cuda_free_wrap = [](sy_img &img) { CHECK(cudaFree(img.data_)) { img.c_ = 0; img.h_ = 0; img.w_ = 0; img.data_ = nullptr; } }; /* a. find fight or fall classification taskId. */ decltype(pThreadParam->TaskInPlayID) interest_task_id; decltype(vptResult) interest_vpt_result; vector interest_imgs(0); int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { const auto &task_id = *_task_id_iter; auto algor_map = pThreadParam->m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::PEDESTRIAN_FIGHT) != algor_map->end() || algor_map->find(algorithm_type_t::PEDESTRIAN_FALL) != algor_map->end()) { interest_task_id.insert(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } /* b. process result. */ if (!interest_imgs.empty()) { std::vector results; pThreadParam->fight_fall_cls_.process_mstreams(interest_task_id, interest_imgs.data(), interest_vpt_result, results); for (auto &result : results) { for (auto &res : result.fight_data) { auto &taskId = res.taskid; auto json_str = helpers::gen_json::gen_pedestrian_fight_json(res, ""); pThreadParam->save_snapshot_process(OBJ_KEY{taskId, (int)*res.objectids.begin()}, algorithm_type_t::PEDESTRIAN_FIGHT, res.ori_img, res.roi_img, res.id, json_str, /* enable_async= */ false); cuda_free_wrap(res.roi_img); } for (auto &res : result.fall_data) { auto &taskId = res.taskid; auto json_str = helpers::gen_json::gen_pedestrian_fall_json(res, ""); pThreadParam->save_snapshot_process(OBJ_KEY{taskId, (int)res.objectid}, algorithm_type_t::PEDESTRIAN_FALL, res.ori_img, res.roi_img, res.id, json_str, false); cuda_free_wrap(res.roi_img); } } } } /* for takeaway member. 外卖员分析模块 */ { /* find takeaway member classification taskId. */ decltype(pThreadParam->TaskInPlayID) interest_task_id; decltype(vptResult) interest_vpt_result; vector interest_imgs(0); int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { const auto &task_id = *_task_id_iter; auto algor_map = pThreadParam->m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::TAKEAWAY_MEMBER_CLASSIFICATION) == algor_map->end()) continue; interest_task_id.insert(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } if (!interest_imgs.empty()) pThreadParam->takeaway_member_.update_mstreams(interest_task_id, interest_imgs.data(), interest_vpt_result); } #endif } } #ifdef WITH_FACE_DET_SS /* for face det 人脸检测抓拍算法模块 */ { #if 0 unsigned image_size = pThreadParam->TaskInPlayID.size(); std::vector facedet_result(image_size); std::vector> face_deleteObjectID(image_size); pThreadParam->m_face_det_ai_engine.ai_engine_process_batch(batch_img, image_size, pThreadParam->TaskInPlayID, facedet_result, face_deleteObjectID); // 跟踪结果送入快照更新 pThreadParam->m_snapshot_reprocessing->update_face_bestsnapshot(pThreadParam->TaskInPlayID, batch_img, facedet_result, face_deleteObjectID); #else decltype(pThreadParam->TaskInPlayID) face_det_interest_task_id; std::vector face_det_interest_imgs(0); int _idx = 0; for (auto _task_id_iter = pThreadParam->TaskInPlayID.begin(); _task_id_iter != pThreadParam->TaskInPlayID.end(); ++_task_id_iter, ++_idx) // loop task_id; { const auto &task_id = *_task_id_iter; if (!task_has_face_algor(task_id)) continue; face_det_interest_task_id.insert(task_id); face_det_interest_imgs.emplace_back(batch_img[_idx]); } if (!face_det_interest_imgs.empty()) { unsigned image_size = face_det_interest_imgs.size(); // 人脸检测、跟踪 std::vector facedet_result(image_size); std::vector> face_deleteObjectID(image_size); pThreadParam->m_face_det_ai_engine.ai_engine_process_batch( face_det_interest_imgs.data(), face_det_interest_imgs.size(), face_det_interest_task_id, facedet_result, face_deleteObjectID); #if 0 // accum int32_t accum = 0; for (const auto &it: facedet_result) accum += it.obj_count; LOG_TRACE(" face result has: {}", accum); #endif // 跟踪结果送入快照更新 pThreadParam->m_snapshot_reprocessing->update_face_bestsnapshot( face_det_interest_task_id, face_det_interest_imgs.data(), facedet_result, face_deleteObjectID); // 保存已结束轨迹的目标 // auto task_iter_face = pThreadParam->TaskInPlayID.begin(); auto task_iter_face = face_det_interest_task_id.begin(); //debug by zsh for (int i = 0; i < face_deleteObjectID.size(); i++) { for (int j = 0; j < face_deleteObjectID[i].size(); ++j) { OBJ_KEY deleteObj = {*task_iter_face, face_deleteObjectID[i][j]}; LOG_TRACE("[info]22222: {}: {}",*task_iter_face,face_deleteObjectID[i][j]); pThreadParam->endframe_obj_process(deleteObj, algorithm_type_t::FACE_SNAPSHOT); } ++task_iter_face; } for (int i = 0; i < face_deleteObjectID.size(); ++i) std::vector().swap(face_deleteObjectID[i]); std::vector>().swap(face_deleteObjectID); std::vector().swap(facedet_result); } #endif } #endif boost::this_thread::sleep(boost::posix_time::milliseconds(1)); ++total_count; } #ifdef WITH_SECOND_PROCESS { std::string msg("taskId to n_frame pair has"); for (auto &iter : task_id_to_n_frame) msg.append(fmt::format("({}: {})", iter.first.c_str(), iter.second)); for (auto &iter : task_id_to_n_frame) { auto &task_id = iter.first; pThreadParam->pedestrian_safety_detector_.force_release_result(task_id); pThreadParam->pedestrian_vehicle_retrograde_.force_release_result(task_id); pThreadParam->takeaway_member_.force_release_result(task_id); } LOG_DEBUG("{}", msg); } #endif // TODO costTime1 += std::chrono::duration_cast(std::chrono::system_clock::now() - begintime1).count(); LOG_INFO("Process Thread is Finished: per frame cost time: {} ms", costTime1); // pThreadParam->m_snaphot_helper.snapShotInfo.clear(); /* while循环退出 ProcessFlag标志位置为false 清空申请资源 */ pThreadParam->ProcessFlag = false; if (pThreadParam->FrameTemp) { CHECK(cudaFree(pThreadParam->FrameTemp)); pThreadParam->FrameTemp = nullptr; } if (batch_img != nullptr) { delete[] batch_img; batch_img = nullptr; } // added by zsh 220801--------------------------------------------------------------------------------- for(auto ss = pThreadParam->system_all_tasks_.begin(); ss != pThreadParam->system_all_tasks_.end(); ) { pThreadParam->system_all_tasks_.erase(ss++); } //----------------------------------------------------------------------------------------------------- return 0; } /* 实现快照保存功能（还未真正保存 将显存图片cp到内存 * 直接保存本地或者存入缓存队列异步保存，保存方式看需求，报警类需要同步保存报警，分析类可异步保存后返回）*/ bool CMultiSourceVideoProcess::save_snapshot_process(const OBJ_KEY &obj_key, const algorithm_type_t &algorithm_type, const sy_img &ori_img, const sy_img &roi_img, const long long id, const std::string &json_str, bool enable_async, const bool ori_img_is_in_gpu, const bool roi_img_is_in_gpu) { auto task_other_params = m_task_param_manager->get_task_other_param(obj_key.video_id); const auto &algor_other_params = task_other_params->find(algorithm_type); if (algor_other_params == task_other_params->end()) { LOG_ERROR("task_id {} not found {} error", obj_key.video_id, int(algorithm_type)); return false; } // TODO const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; // modified by zsh 图片名添加时间戳---------------------------------------------------------------------------------------- std::string cur_timestamp_ms = std::to_string(helpers::timer::get_timestamp()); const std::string fpath_ori = basic_param->result_folder + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + std::to_string(id) + "_" + cur_timestamp_ms + ".jpg"; const std::string fpath_roi = basic_param->result_folder_little + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + std::to_string(id) + "_" + cur_timestamp_ms + ".jpg"; //------------------------------------------------------------------------------------------------------------------------ multi_image_t m_image(ori_img, roi_img); { m_image.ori_fpath = fpath_ori; m_image.roi_fpath = fpath_roi; m_image.ori_data_device = ori_img_is_in_gpu ? data_device_t::GPU : data_device_t::CPU; m_image.roi_data_device = roi_img_is_in_gpu ? data_device_t::GPU : data_device_t::CPU; } // 调用快照保存后处理模块 将快照保存 m_save_snapshot_reprocessing->reprocessing_process_wo_locus(obj_key, m_image, enable_async #ifdef POST_USE_RABBITMQ , json_str #endif ); return true; } /* 每帧都需要做的算法模块 */ int CMultiSourceVideoProcess::everyframe_process(set &task_in_play_id, sy_img *images, vector &ol_det_result) { #ifdef WITH_SECOND_PROCESS /* 人数聚集算法功能 每帧都会获取算法结果 并返回 */ auto results = m_human_gather_statistics.human_gather_statistics_process(task_in_play_id, images, ol_det_result); for (auto &result : results) { #ifdef POST_USE_RABBITMQ auto json_str = helpers::gen_json::gen_human_gather_json(result.task_id, result.boxes, ""); #endif auto task_id = result.task_id; auto task_other_params = m_task_param_manager->get_task_other_param(task_id); const auto &algor_other_params = task_other_params->find(algorithm_type_t::HUMAN_GATHER); if (algor_other_params == task_other_params->end()) { LOG_ERROR("[Error] taskId {} not found algor {}", task_id.c_str(), (int)algorithm_type_t::HUMAN_GATHER); continue; } const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; // modified by zsh 图片名添加时间戳---------------------------------------------------------------------------------------- std::string cur_timestamp_ms = std::to_string(helpers::timer::get_timestamp()); const std::string fpath_ori = basic_param->result_folder + helpers::os::sep + task_id + "_" + std::to_string(result.boxes.size()) + "_" + std::to_string(result.id) + "_" + cur_timestamp_ms + ".jpg"; //------------------------------------------------------------------------------------------------------------------------ multi_image_t m_image(*result.img, sy_img()); { m_image.ori_fpath = fpath_ori; m_image.ori_data_device = data_device_t::GPU; } OBJ_KEY obj_key{task_id, 0}; m_save_snapshot_reprocessing->reprocessing_process_wo_locus(obj_key, m_image, true #ifdef POST_USE_RABBITMQ , json_str #endif ); } #endif return 0; } /* 轨迹结束帧需要做的算法模块 */ int CMultiSourceVideoProcess::endframe_obj_process(const OBJ_KEY &obj_key, algorithm_type_t algor_type) { auto task_param_ptr = m_task_param_manager->get_task_algor_param(obj_key.video_id); auto task_other_param_ptr = m_task_param_manager->get_task_other_param(obj_key.video_id); // 该路任务开启了抓拍功能 开始抓拍保存；若未开启抓拍，清空显存资源 if ((task_param_ptr->human_algors.find(algorithm_type_t::HUMAN_SNAPSHOT) != task_param_ptr->human_algors.end() || task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_SNAPSHOT) != task_param_ptr->vehicle_algors.end() || task_param_ptr->nonmotor_vehicle_algors.find(algorithm_type_t::NONMOTOR_VEHICLE_SNAPSHOT) != task_param_ptr->nonmotor_vehicle_algors.end())) { #ifdef MTASK_DEBUG_ LOG_DEBUG("task_id {} endframe_obj_process obj_id {}", obj_key.video_id, obj_key.obj_id); #endif m_save_snapshot_reprocessing->reprocessing_process(obj_key); } else { // printf("delete snapshot\n"); m_snapshot_reprocessing->delete_finishtask_snapshot(obj_key.video_id, obj_key.obj_id); } #ifdef WITH_SECOND_PROCESS /* 开启外卖员算法模块，获取该目标的算法分析结果 返回结果+快照 最后释放资源 */ if (task_param_ptr->nonmotor_vehicle_algors.find(algorithm_type_t::TAKEAWAY_MEMBER_CLASSIFICATION) != task_param_ptr->nonmotor_vehicle_algors.end()) { auto result = takeaway_member_.get_result_by_objectid(ai_engine_module::obj_key_t{ obj_key.obj_id, obj_key.video_id, algorithm_type_t::TAKEAWAY_MEMBER_CLASSIFICATION}); if (result.get()) { auto json_str = helpers::gen_json::gen_takeaway_member_cls_json(obj_key.video_id, obj_key.obj_id, result->box, result->category); save_snapshot_process(obj_key, algorithm_type_t::TAKEAWAY_MEMBER_CLASSIFICATION, result->ori_img, result->roi_img, 0, json_str); CHECK(cudaFree((void *)(result->roi_img.data_))); CHECK(cudaFree((void *)(result->ori_img.data_))); } } /* 开启行人&机动车逆行算法模块，获取该目标的算法分析结果 返回结果+快照 最后释放资源 */ if (task_param_ptr->human_algors.find(algorithm_type_t::PEDESTRIAN_RETROGRADE) != task_param_ptr->human_algors.end() || task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_RETROGRADE) != task_param_ptr->vehicle_algors.end()) { auto func_wrap = [&](const algorithm_type_t &algor_type) { auto results = pedestrian_vehicle_retrograde_.get_results_by_id( ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algor_type}); if (results) { bool flag_pRetroGrade{true}, flag_vRetroGrade{true}; // modified by zsh for (unsigned idx = 0; idx < results->size(); ++idx) { auto &result = results->at(idx); // modified by zsh 一个id一种事件只报警一次------------------------------------------------------------------ if(algor_type==algorithm_type_t::PEDESTRIAN_RETROGRADE && flag_pRetroGrade == true) { auto &&json_str = helpers::gen_json::gen_retrograde_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/idx, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); flag_pRetroGrade = false; } if(algor_type==algorithm_type_t::VEHICLE_RETROGRADE && flag_vRetroGrade == true) { auto &&json_str = helpers::gen_json::gen_retrograde_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/idx, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); flag_vRetroGrade = false; } // ------------------------------------------------------------------------------------------------------- if(0) { auto &&json_str = helpers::gen_json::gen_retrograde_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/idx, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); } if (result.roi_img.data_ != nullptr) { if (result.roi_img_is_in_gpu) { LOG_DEBUG("free roi gpu memory."); CHECK(cudaFree((void *)(result.roi_img.data_))) } result.roi_img.data_ = nullptr; } if (result.ori_img.data_ != nullptr) { if (result.ori_img_is_in_gpu) { LOG_DEBUG("free ori gpu memory."); CHECK(cudaFree((void *)(result.ori_img.data_))) } result.ori_img.data_ = nullptr; } } } }; if (task_param_ptr->human_algors.find(algorithm_type_t::PEDESTRIAN_RETROGRADE) != task_param_ptr->human_algors.end()) func_wrap(algorithm_type_t::PEDESTRIAN_RETROGRADE); if (task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_RETROGRADE) != task_param_ptr->vehicle_algors.end()) func_wrap(algorithm_type_t::VEHICLE_RETROGRADE); } /* 开启行人&机动车非法闯入算法模块，获取该目标的算法分析结果 返回结果+快照 最后释放资源 */ if (task_param_ptr->human_algors.find(algorithm_type_t::PEDESTRIAN_TRESPASS) != task_param_ptr->human_algors.end() || task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_TRESPASS) != task_param_ptr->vehicle_algors.end()) { auto func_wrap = [&](const algorithm_type_t &algor_type) { auto results = pedestrian_vehicle_trespass_.get_results_by_id( ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algor_type}); if (results) { bool flag_pThePass{true}, flag_vThePass{true}; // modified by zsh for (unsigned idx = 0; idx < results->size(); ++idx) { auto &result = results->at(idx); // modified by zsh 一个id一种事件只报警一次------------------------------------------------------------------ if(algor_type==algorithm_type_t::PEDESTRIAN_TRESPASS && flag_pThePass == true) { auto &&json_str = helpers::gen_json::gen_retrograde_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/idx, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); flag_pThePass = false; } if(algor_type==algorithm_type_t::VEHICLE_TRESPASS && flag_vThePass == true) { auto &&json_str = helpers::gen_json::gen_retrograde_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/idx, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); flag_vThePass = false; } // ------------------------------------------------------------------------------------------------------- if(0) { auto &&json_str = helpers::gen_json::gen_retrograde_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); // printf("%s\n", json_str.c_str()); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/idx, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); } if (result.roi_img.data_ != nullptr) { if (result.roi_img_is_in_gpu) CHECK(cudaFree((void *)(result.roi_img.data_))) result.roi_img.data_ = nullptr; } if (result.ori_img.data_ != nullptr) { if (result.ori_img_is_in_gpu) CHECK(cudaFree((void *)(result.ori_img.data_))) result.ori_img.data_ = nullptr; } } } }; if (task_param_ptr->human_algors.find(algorithm_type_t::PEDESTRIAN_TRESPASS) != task_param_ptr->human_algors.end()) func_wrap(algorithm_type_t::PEDESTRIAN_TRESPASS); if (task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_TRESPASS) != task_param_ptr->vehicle_algors.end()) func_wrap(algorithm_type_t::VEHICLE_TRESPASS); } /* 开启行人安全分析算法模块，获取该目标的算法分析结果 返回结果+快照 最后释放资源 */ if (task_param_ptr->human_algors.find(algorithm_type_t::NO_REFLECTIVE_CLOTHING) != task_param_ptr->human_algors.end() || task_param_ptr->human_algors.find(algorithm_type_t::NO_SAFETY_HELMET) != task_param_ptr->human_algors.end() || task_param_ptr->human_algors.find(algorithm_type_t::CALL_PHONE_DET) != task_param_ptr->human_algors.end() || task_param_ptr->human_algors.find(algorithm_type_t::SMOKING_DET) != task_param_ptr->human_algors.end()) { LOG_TRACE("get result task_id {} obj_id {}", obj_key.video_id, obj_key.obj_id); auto results = pedestrian_safety_detector_.get_results_by_id( ai_engine_module::unique_obj_id_t{obj_key.obj_id, obj_key.video_id}); // auto results = std::shared_ptr(nullptr); if (results) { //! loop per pedestrian safety attribute. long id = 0; bool flag_nReflect{true}, flag_nHelmet{true}, flag_cPhone{true}, flag_smoking{true}; // modified by zsh LOG_DEBUG(" size of results is {}", results->size()); for (auto result : *results) { for (auto algor_type_iter = result.algorithm_type_seq.begin(); algor_type_iter != result.algorithm_type_seq.end(); ++algor_type_iter) { auto algor_type = *algor_type_iter; // modified by zsh 一个id一种事件只报警一次------------------------------------------------------------------ if(algor_type==algorithm_type_t::NO_REFLECTIVE_CLOTHING && flag_nReflect == true) { auto json_str = helpers::gen_json::gen_pedestrian_safety_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/id++, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); flag_nReflect = false; LOG_TRACE("PEDESTRIAN task_id {} obj_id {} algor_type {}", obj_key.video_id, obj_key.obj_id, int(algor_type)); } if(algor_type==algorithm_type_t::NO_SAFETY_HELMET && flag_nHelmet == true) { auto json_str = helpers::gen_json::gen_pedestrian_safety_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/id++, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); flag_nHelmet = false; LOG_TRACE("PEDESTRIAN task_id {} obj_id {} algor_type {}", obj_key.video_id, obj_key.obj_id, int(algor_type)); } if(algor_type==algorithm_type_t::CALL_PHONE_DET && flag_cPhone == true) { auto json_str = helpers::gen_json::gen_pedestrian_safety_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/id++, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); flag_cPhone = false; LOG_TRACE("PEDESTRIAN task_id {} obj_id {} algor_type {}", obj_key.video_id, obj_key.obj_id, int(algor_type)); } if(algor_type==algorithm_type_t::SMOKING_DET && flag_smoking == true) { auto json_str = helpers::gen_json::gen_pedestrian_safety_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/id++, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); flag_smoking = false; LOG_TRACE("PEDESTRIAN task_id {} obj_id {} algor_type {}", obj_key.video_id, obj_key.obj_id, int(algor_type)); } // ------------------------------------------------------------------------------------------------------- if(0) { LOG_TRACE("PEDESTRIAN task_id {} obj_id {} algor_type {}", obj_key.video_id, obj_key.obj_id, int(algor_type)); auto json_str = helpers::gen_json::gen_pedestrian_safety_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, result.ori_img, result.roi_img, /*id=*/id++, json_str, true, result.ori_img_is_in_gpu, result.roi_img_is_in_gpu); } } if (result.roi_img.data_ != nullptr) { if (result.roi_img_is_in_gpu) CHECK(cudaFree((void *)(result.roi_img.data_))); result.roi_img.data_ = nullptr; } if (result.ori_img.data_ != nullptr) { if (result.ori_img_is_in_gpu) CHECK(cudaFree((void *)(result.ori_img.data_))); result.ori_img.data_ = nullptr; } } } } #endif /* 开启人脸抓拍分析算法模块，获取该目标的算法分析结果 返回结果+快照 最后释放资源 */ if (task_other_param_ptr->find(algorithm_type_t::FACE_SNAPSHOT) != task_other_param_ptr->end() && algor_type == algorithm_type_t::FACE_SNAPSHOT) { LOG_TRACE("--- face snapshot will save snapshot IAMGE, task: {} obj: {}.", obj_key.video_id, obj_key.obj_id); m_save_snapshot_reprocessing->reprocessing_process_face(obj_key); } return 0; } /* 结束解码器并释放资源 */ bool CMultiSourceVideoProcess::FinishDecode(std::pair &iter) { boost::thread::sleep(boost::get_system_time() + boost::posix_time::milliseconds(400)); { task_resource &tmp_task = iter.second; tmp_task.task_state == FINISH; tmp_task.taskcuvid->DxCloseDecoder(); delete tmp_task.taskcuvid; tmp_task.taskcuvid = nullptr; } return true; } /* 结束解码器并释放资源 */ void CMultiSourceVideoProcess::FinishDecode(const string taskID) { boost::thread::sleep(boost::get_system_time() + boost::posix_time::milliseconds(400)); for (const auto &iter : system_all_tasks_) { if (iter.first == taskID && system_all_tasks_[iter.first].taskcuvid != nullptr) { task_resource &tmp_task = system_all_tasks_[iter.first]; tmp_task.task_state == FINISH; tmp_task.taskcuvid->DxCloseDecoder(); delete tmp_task.taskcuvid; tmp_task.taskcuvid = nullptr; LOG_INFO("finish task: {:s}", taskID.c_str()); break; } } } /* 带超时等待的结束任务接口 */ int CMultiSourceVideoProcess::WaitAndFinishTask(const string taskID, const int max_timeout_ms) { int error_code = FAILED; std::unique_lock lk(taskMutex); AddOperator(taskID, FINISHTASK); taskCondVar.wait_for(lk, std::chrono::milliseconds(max_timeout_ms)); if (system_all_tasks_.count(taskID)) { task_resource &tmp_task = system_all_tasks_[taskID]; error_code = tmp_task.task_state == FINISH ? SUCCESS : FAILED; } if (SUCCESS != error_code) // if (!DeleteTaskQ(taskID)) if (DeleteTaskQ(taskID)) // modifed by zsh 220729 LOG_ERROR("finish {} failed", taskID.c_str()); return error_code; } /* 带超时等待的暂停任务接口 */ int CMultiSourceVideoProcess::WaitAndPauseTask(const string taskID, const int max_timeout_ms) { int error_code = FAILED; std::unique_lock lk(taskMutex); AddOperator(taskID, PAUSETASK); taskCondVar.wait_for(lk, std::chrono::milliseconds(max_timeout_ms)); if (system_all_tasks_.count(taskID)) { task_resource &tmp_task = system_all_tasks_[taskID]; error_code = tmp_task.task_state == PAUSE ? SUCCESS : FAILED; } if (SUCCESS != error_code) // if (!DeleteTaskQ(taskID)) if (DeleteTaskQ(taskID)) // modifed by zsh 220729 LOG_ERROR("pause {} failed", taskID.c_str()); return error_code; } /* 带超时等待的重启任务接口 */ int CMultiSourceVideoProcess::WaitAndRestartTask(const string taskID, const int max_timeout_ms) { int error_code = FAILED; std::unique_lock lk(taskMutex); AddOperator(taskID, RESTARTTASK); taskCondVar.wait_for(lk, std::chrono::milliseconds(max_timeout_ms)); if (system_all_tasks_.count(taskID)) { task_resource &tmp_task = system_all_tasks_[taskID]; error_code = tmp_task.task_state == PLAY ? SUCCESS : FAILED; } if (SUCCESS != error_code) // if (!DeleteTaskQ(taskID)) if (DeleteTaskQ(taskID)) // modifed by zsh 220729 LOG_ERROR("restart {} failed", taskID.c_str()); return error_code; } /* 结束任务接口 */ void CMultiSourceVideoProcess::FinishTask(const string taskID, const bool delete_snapshot) { if (!system_all_tasks_.count(taskID)) { LOG_ERROR("finish task: {} failed (task id error)", taskID.c_str()); return; } task_resource &tmp_task = system_all_tasks_[taskID]; // added by zsh 220801 针对已finish但未从system_all_tasks_删除的任务 if (tmp_task.task_state == FINISH) { LOG_ERROR("finish task: {} failed (task has already been finished.)", taskID.c_str()); return; } if (tmp_task.task_state == PLAY) TaskInPlay--; tmp_task.task_state = FINISH; task_id_to_processed_frame_[taskID] = 0; // m_snaphot_helper.finish_task_ss_analysis(taskID, m_hp_analysis_config, m_hcp_analysis_config, // m_vehicle_analysis_config, m_hf_recg_config, m_hcf_recg_config, m_vcf_recg_config); //是否开启车辆特征识刿; FinishTaskTracker(VPT_Handle_, taskID); #ifdef WITH_FACE_DET_SS // 人脸任务结束 auto task_param_ptr = m_task_param_manager->get_task_algor_param(taskID); if (task_param_ptr->human_face_algors.find(algorithm_type_t::FACE_SNAPSHOT) != task_param_ptr->human_face_algors.end()) { m_face_det_ai_engine.operator_tracker(taskID, FINISHTASK, 0); m_face_det_ai_engine.finish_task(taskID); } #endif if (tmp_task.task_algorithm_data.frame != nullptr) { CHECK(cudaFree(tmp_task.task_algorithm_data.frame)); tmp_task.task_algorithm_data.frame = nullptr; } m_task_param_manager->delete_task_param(taskID); // TODO: ??? if (delete_snapshot) { m_snapshot_reprocessing->delete_finishtask_snapshot(taskID); ((save_snapshot_reprocessing *)m_save_snapshot_reprocessing)->delete_finishtask(taskID); } if (strcmp(taskID.c_str(), viewTaskID.c_str()) == 0) viewTaskID = ""; LOG_INFO("finish task: {}", taskID.c_str()); } /* 暂停任务接口 */ void CMultiSourceVideoProcess::PauseTask(const string taskID) { if (!system_all_tasks_.count(taskID)) { LOG_ERROR("pause task: {} failed (task id error)", taskID.c_str()); return; } task_resource &tmp_task = system_all_tasks_[taskID]; // added by zsh 220801 针对已finish但未从system_all_tasks_删除的任务 if (tmp_task.task_state == FINISH) { LOG_ERROR("pause task: {} failed (task has already been finished.)", taskID.c_str()); return; } if (tmp_task.task_state == PLAY) TaskInPlay--; tmp_task.task_state = PAUSE; task_id_to_processed_frame_[taskID] = 0; PauseTaskTracker(VPT_Handle_, taskID); #ifdef WITH_FACE_DET_SS // 人脸任务暂停 auto task_param_ptr = m_task_param_manager->get_task_algor_param(taskID); if (task_param_ptr->human_face_algors.find(algorithm_type_t::FACE_SNAPSHOT) != task_param_ptr->human_face_algors.end()) { m_face_det_ai_engine.operator_tracker(taskID, PAUSETASK, 0); } #endif tmp_task.taskcuvid->PauseDecoder(); if (strcmp(taskID.c_str(), viewTaskID.c_str()) == 0) viewTaskID = ""; LOG_INFO("pause task: {}", taskID.c_str()); } /* 重启任务接口 */ void CMultiSourceVideoProcess::RestartTask(const string taskID) { if (!system_all_tasks_.count(taskID)) { LOG_ERROR("restart task: {} failed (task id error)", taskID.c_str()); return; } task_resource &tmp_task = system_all_tasks_[taskID]; // added by zsh 220801 针对已finish但未从system_all_tasks_删除的任务 if (tmp_task.task_state == FINISH) { LOG_ERROR("restart task: {} failed (task has already been finished.)", taskID.c_str()); return; } tmp_task.task_state = PLAY; TaskInPlay++; RestartTaskTracker(VPT_Handle_, taskID); #ifdef WITH_FACE_DET_SS // 人脸任务重启 auto task_param_ptr = m_task_param_manager->get_task_algor_param(taskID); if (task_param_ptr->human_face_algors.find(algorithm_type_t::FACE_SNAPSHOT) != task_param_ptr->human_face_algors.end()) { m_face_det_ai_engine.operator_tracker(taskID, RESTARTTASK, 0); } #endif tmp_task.taskcuvid->ResumeDecoder(); LOG_INFO("restart task: {}", taskID.c_str()); } bool CMultiSourceVideoProcess::DeleteTaskQ(const string taskID) { // std::lock_guard lk(taskMutex); // modifed by zsh 220729 for (auto it = TaskOperatorQ.begin(); it != TaskOperatorQ.end();) { if (it->changeTaskID == taskID) { TaskOperatorQ.erase(it); return true; } else ++it; } return false; } /* 添加任务接口 */ bool CMultiSourceVideoProcess::add_task_operation(task_param _cur_task_param) { /* 通过ProcessFlag标志位来判断 是否是初次启动，若为初次启动，申请必要资源 开启主线程 */ if (!ProcessFlag) DxDecoderInterface::InitDecoderModule(); /* 给新任务创建解码器 */ /* step1.判断流是否支持解码 */ int ret = DxDecoderInterface::IsSupport(_cur_task_param.ipc_url); if (0 != ret) { LOG_ERROR("not support codec {}.", ret); return false; } task_resource &new_add_task_param = system_all_tasks_[_cur_task_param.task_id]; /* step2.创建新的解码器 */ DxConfig cfg = {0}; cfg.devId = gpu_id_; cfg.decMode = 0; cfg.colorFmt = 0; // cfg.forceTcp = false; cfg.forceTcp = true; cfg.type = DX_DECODER_TYPE_CUDA; new_add_task_param.taskcuvid = new DxDecoderWrap(&cfg); if (nullptr == new_add_task_param.taskcuvid) { LOG_ERROR("Add New DxDecoder Failed!"); system_all_tasks_.erase(_cur_task_param.task_id); AddTaskSucFlag = FAILED; return false; } /* step3.打开解码器 取一帧数据 获取流宽高 用于非法闯入功能初始化mask */ int input_image_width = 0, input_image_height = 0; if (new_add_task_param.taskcuvid->DxOpenDecoder(_cur_task_param.ipc_url, SKIP_FRAME) != 0) { LOG_ERROR("Add Task Failed! Please check you video file name!"); delete new_add_task_param.taskcuvid; new_add_task_param.taskcuvid = nullptr; system_all_tasks_.erase(_cur_task_param.task_id); AddTaskSucFlag = VIDEOFILEERROR; return false; } else { DxGPUFrame frame = {}; if (new_add_task_param.taskcuvid->DxDecoderIsRun()) { while (1) { if (new_add_task_param.taskcuvid->DxLockFrame(&frame) == 0) { input_image_width = frame.width; input_image_height = frame.height; new_add_task_param.taskcuvid->DxUnlockFrame(); break; } } } } /* 第一次这里添加任务参数 后续在AddOperator里面直接添加 */ if (!ProcessFlag) m_task_param_manager->add_task_param(_cur_task_param.task_id, _cur_task_param); #ifdef WITH_SECOND_PROCESS /* 如果开启了行人 机动车非法闯入功能 生成闯入区域mask */ auto new_task_algor_param = m_task_param_manager->get_task_other_param(_cur_task_param.task_id); if (new_task_algor_param->find(algorithm_type_t::PEDESTRIAN_TRESPASS) != new_task_algor_param->end()) { pedestrian_vehicle_trespass_.pedestrianvehicletrespass_init_region( _cur_task_param.task_id, algorithm_type_t::PEDESTRIAN_TRESPASS, input_image_width, input_image_height); } if (new_task_algor_param->find(algorithm_type_t::VEHICLE_TRESPASS) != new_task_algor_param->end()) { pedestrian_vehicle_trespass_.pedestrianvehicletrespass_init_region( _cur_task_param.task_id, algorithm_type_t::VEHICLE_TRESPASS, input_image_width, input_image_height); } #endif new_add_task_param.task_state = PLAY; /* 第一次添加任务 开启算法主线程 */ if (!ProcessFlag) { LOG_DEBUG("open ThreadProcess"); ProcessThread = boost::thread(ThreadProcess, this); ProcessFlag = true; } ((save_snapshot_reprocessing *)m_save_snapshot_reprocessing)->add_newtask(_cur_task_param.task_id); LOG_INFO("add task: {}", _cur_task_param.task_id); AddTaskSucFlag = SUCCESS; return true; } #ifdef POST_USE_RABBITMQ /* MQ队列的初始化 */ int CMultiSourceVideoProcess::AddMqConn(mq_type_t mq_type, rabbitmq_conn_params_t mq_conn_param) { /* 初始化MQ队列 */ if (!mq_manager_->add_conn(mq_type, mq_conn_param)) { LOG_ERROR("Connection MQ failed, ip: {} port: {} uname: {} passwd: {}", mq_conn_param.ip, mq_conn_param.port, mq_conn_param.uname, mq_conn_param.passwd); return MQ_CONN_ERROR; } /* 为报警类 绑定回调 传入mq_manager_.publish 内部直接调用*/ if (mq_type_t::ALARM_MQ == mq_type) m_save_snapshot_reprocessing->set_callback( std::bind(&mq::Manager::publish, mq_manager_, mq_type, std::placeholders::_1, true)); return SUCCESS; } /* 获取任务的状态 MQ返回 */ int CMultiSourceVideoProcess::GetTaskStatus(const string taskID) { std::vector taskids; std::vector statues; for (auto iter = system_all_tasks_.begin(); iter != system_all_tasks_.end(); ++iter) { if (taskID.empty() || iter->first == taskID) { int status = 0; LOG_TRACE("task_state: {}: {}",iter->first,iter->second.task_state); // debug by zsh switch (iter->second.task_state) { case PLAY: status = 1; break; case PAUSE: status = 2; break; } taskids.emplace_back(iter->first); statues.emplace_back(status); } } if (!taskids.empty()) { auto json_str = helpers::gen_json::gen_task_status_json(taskids, statues); // mq_manager_->publish(mq_type_t::GET_TASK_MQ, json_str.c_str()); mq_manager_->publish(mq_type_t::GET_TASK_MQ, json_str.c_str(),true); } return SUCCESS; } #endif /* 添加任务操作 此处的添加 指添加任务的添加 */ int CMultiSourceVideoProcess::AddOperator(task_param tparam) { if (!ProcessFlag) { LOG_INFO("==> begin first add"); auto ret = add_task_operation(tparam); boost::thread::sleep(boost::get_system_time() + boost::posix_time::microseconds(500)); return ret == true ? SUCCESS : FAILED; } else { std::unique_lock l(taskMutex); // 先做算法参数初始化 m_task_param_manager->add_task_param(tparam.task_id, tparam); AddTaskSucFlag = 1; // 待处理 Operator newOper; newOper.changeTaskID = tparam.task_id; newOper.changeTaskOperator = ADDTASK; newOper.videoFileName = tparam.ipc_url; newOper.resultFolderLittleName = nullptr; newOper.resultFolderName = nullptr; TaskOperatorQ.push_back(newOper); taskCondVar.wait_for(l, std::chrono::seconds(20)); } int addRes = FAILED; if (AddTaskSucFlag == SUCCESS) // 添加成功 { addRes = SUCCESS; } else if (AddTaskSucFlag < SUCCESS) // 添加失败 请求已处理 { addRes = AddTaskSucFlag; } else if (AddTaskSucFlag == 1) // 超时 一直为处理这条请求 { if (!TaskOperatorQ.empty()) { Operator newOper = TaskOperatorQ.back(); if (strcmp(newOper.videoFileName, tparam.ipc_url) == 0) { TaskOperatorQ.pop_back(); } LOG_ERROR("Failed Add New Task! Algorithm Process Error!"); } addRes = TIMEOUT_ERROR; } return addRes; } /* 添加对任务的操作至缓存队列 此处的添加 指添加任务的（暂停、重启、结束）的添加，等待while结束后集中处理 */ void CMultiSourceVideoProcess::AddOperator(string taskID, int taskOper) { if (taskOper > 0 && taskOper < 4) { Operator newOper = {}; newOper.changeTaskID = taskID; newOper.changeTaskOperator = TaskOperator(taskOper); newOper.videoFileName = nullptr; TaskOperatorQ.push_back(newOper); } } /* 任务操作缓存队列 真正操作对任务的修改 */ void CMultiSourceVideoProcess::OperatorTask() { for (const auto &cur_task : system_all_tasks_) { if ((cur_task.second.task_state == PLAY || cur_task.second.task_state == DECODEERROR)) { if (!cur_task.second.taskcuvid->DxDecoderIsRun()) { LOG_INFO("decoder can not run: task id: {} task state: {}", cur_task.first.c_str(), cur_task.second.task_state); FinishTask(cur_task.first, false); FinishDecode(cur_task.first); #ifdef POST_USE_RABBITMQ auto json_str = helpers::gen_json::gen_office_task_heart_beat_json({cur_task.first}); mq_manager_->publish(mq_type_t::HEART_BEAT_MQ, json_str.c_str(), true); #endif } } } while (!TaskOperatorQ.empty()) { Operator newOperator = TaskOperatorQ.front(); TaskOperatorQ.pop_front(); switch (newOperator.changeTaskOperator) { case ADDTASK: task_param tparam; tparam.task_id = newOperator.changeTaskID.c_str(); tparam.ipc_url = newOperator.videoFileName; tparam.algor_counts = newOperator.algor_counts; add_task_operation(tparam); break; case PAUSETASK: PauseTask(newOperator.changeTaskID); break; case RESTARTTASK: RestartTask(newOperator.changeTaskID); break; case FINISHTASK: { LOG_DEBUG("{} start Finish Task.", newOperator.changeTaskID); FinishTask(newOperator.changeTaskID, true); } break; default: break; } } } /* 授权校验线程 */ #ifdef AUTHORIZATION void check_thread(void *handle) { int res = -1; #ifndef _MSC_VER char wtime[15]; memset(wtime, 0, 15); char *time = wtime; #endif CMultiSourceVideoProcess *pThreadParam = (CMultiSourceVideoProcess *)handle; while (1) { #ifdef _MSC_VER res = sy_licence(productSN); #else res = sy_licence(productSN, &time); #endif if (res < 0) { pThreadParam->licence_status = pThreadParam->licence_status - 1; } else { if (pThreadParam->licence_status < 0) { pThreadParam->licence_status = 0; } } boost::this_thread::sleep(boost::posix_time::seconds(300)); // 5min } } #endif