#include "MultiSourceProcess.h" #include "../common/logger.hpp" #include #include #include #include #include #include "../decoder/interface/DecoderManager.h" #include "../decoder/interface/utiltools.hpp" #include "../helpers/time_helper.hpp" #include "../helpers/os_helper.hpp" #include "../helpers/gen_json.hpp" #include "../helpers/str_helper.hpp" #include "../reprocessing_module/save_snapshot_reprocessing.h" #include "macro_definition.h" #include "ErrorInfo.h" #include "SourceSingleton.hpp" #include "mvpt_process_assist.h" #include "../util/vpc_util.h" #include "../decoder/gb28181/websocket/WebsocketClient.h" // #define VEHICLE_MULTI_BOXES #define WITH_FACE_DET_SS #define productSN "51C4B28135604F649671727185949A91" //linux 通途抓拍引擎产品序列号 using namespace std; map index_to_algo_type = {{0, algorithm_type_t::HUMAN_SNAPSHOT}, {1, algorithm_type_t::NONMOTOR_VEHICLE_SNAPSHOT}, {2, algorithm_type_t::NONMOTOR_VEHICLE_SNAPSHOT}, {3, algorithm_type_t::NONMOTOR_VEHICLE_SNAPSHOT}, {4, algorithm_type_t::VEHICLE_SNAPSHOT}, {5, algorithm_type_t::VEHICLE_SNAPSHOT}, {6, algorithm_type_t::VEHICLE_SNAPSHOT}, {7, algorithm_type_t::VEHICLE_SNAPSHOT}, {8, algorithm_type_t::VEHICLE_SNAPSHOT}, {9, algorithm_type_t::FACE_SNAPSHOT}}; struct decode_cbk_userdata{ string task_id; void* opaque; void* opaque1; }; /** * 注意: gpuFrame 在解码器设置的显卡上,后续操作要十分注意这一点,尤其是多线程情况 * */ void post_decod_cbk(const void * userPtr, DeviceMemory* devFrame){ decode_cbk_userdata* ptr = (decode_cbk_userdata*)userPtr; if (ptr!= nullptr) { CMultiSourceProcess* _this = (CMultiSourceProcess*)ptr->opaque; if(nullptr != _this){ _this->decoded_cbk(devFrame); } } } void decode_finished_cbk(const void * userPtr){ decode_cbk_userdata* ptr = (decode_cbk_userdata*)userPtr; if (ptr!= nullptr) { CMultiSourceProcess* _this = (CMultiSourceProcess*)ptr->opaque; if(nullptr != _this){ _this->task_finished(ptr->task_id); } } delete ptr; ptr = nullptr; } CMultiSourceProcess::CMultiSourceProcess(){ m_timing_snapshot_thread = nullptr; } CMultiSourceProcess::~CMultiSourceProcess(){ } int CMultiSourceProcess::InitAlgorthim(tsl_aiplatform_param vptParam){ set_default_logger(LogLevel(vptParam.log_level), "multi_source_process", vptParam.log_path, vptParam.log_mem, vptParam.log_mem); LOG_INFO("编译时间:{} {}", __DATE__, __TIME__); #ifdef USE_VILLAGE if (!CheckTime()) { //时间限制 LOG_ERROR("sy_licence_check failed."); return AUTHOR_ERROR; } #else if (!CheckLabel(vptParam.gpuid)) { //机器授权 LOG_ERROR("CheckLabel failed."); return AUTHOR_ERROR; } #endif SourceSingleton::getInstance(); skip_frame_ = 5; m_batch_size = 20; m_devId = vptParam.gpuid; string models_dir = vptParam.models_dir; VPTProcess_PARAM vparam; vparam.gpuid = m_devId; vparam.max_batch = m_batch_size; vparam.threshold = 0.4; vparam.model_dir = models_dir; aclrtSetDevice(m_devId); int ret = vpt_process.init(vparam); if (ret < 0){ return ret; } #ifdef USE_VILLAGE //道路分割 if(seg_process.init(vptParam.gpuid, models_dir) != 0) { LOG_FATAL("Init road_seg failed"); return -1; } if(seg3cls_process.init(vptParam.gpuid, models_dir) != 0) { LOG_FATAL("Init road_3clsseg failed"); return -1; } //三轮车头肩检测 if (!tricycle_manned_.init(vptParam.gpuid, models_dir)) { LOG_FATAL("Init tricycle_hs failed"); return -1; } //货车头肩检测 if (!truck_manned_.init(vptParam.gpuid, models_dir)) { LOG_FATAL("Init truck_hs failed"); return -1; } //二轮车头肩检测 if (!motor_hsprocess_.init(vptParam.gpuid, models_dir)) { LOG_FATAL("Init motor_hs failed"); return -1; } //二轮车使用手机及加雨棚 if (!motor_refit_phoneprocess_.init(vptParam.gpuid, models_dir)) { LOG_FATAL("Init motor_phone failed"); return -1; } /* //二轮车使用手机检测 if (!motor_phoneprocess_.init(vptParam.gpuid, models_dir)) { LOG_FATAL("Init motor_phone failed"); return -1; }*/ //电动车改装(加雨棚) if (!motor_refitprocess_.init(vptParam.gpuid, models_dir)) { LOG_FATAL("Init motor_refit failed"); return -1; } // 行人/二轮车闯红灯 if (!traffic_lightprocess_.init(vptParam.gpuid, models_dir)) { LOG_FATAL("Init traffic_light failed"); return -1; } // 与道路分割相关的算法 if (!roadseg_corrprocess_.init(vptParam.gpuid, models_dir)) { LOG_FATAL("Init roadseg_corrprocess failed"); return -1; } #endif #ifdef WITH_FACE_DET_SS // 人脸检测初始化 facedet_ai_engine_param fd_param; char model_path_yolov5s[100]; strcpy(model_path_yolov5s, (models_dir + "/models/face_detect/face_det_yolov5s_310p.om").c_str()); fd_param.sdk_param.det_modelNames = model_path_yolov5s; char model_path_ldmk[100]; strcpy(model_path_ldmk, (models_dir + "/models/face_detect/face_ldmk_310p.om").c_str()); fd_param.sdk_param.ldmk_modelNames = model_path_ldmk; char model_path_pose[100]; strcpy(model_path_pose, (models_dir + "/models/face_detect/face_pose_310p.om").c_str()); fd_param.sdk_param.pose_modelNames = model_path_pose; char model_path_score[100]; strcpy(model_path_score, (models_dir + "/models/face_detect/face_score_310p.om").c_str()); fd_param.sdk_param.score_modelNames = model_path_score; char model_path_fuzzy[100]; strcpy(model_path_fuzzy, (models_dir + "/models/face_detect/face_fuzzy_310p.om").c_str()); fd_param.sdk_param.fuzzy_modelNames = model_path_fuzzy; char model_path_occlusion[100]; strcpy(model_path_occlusion, (models_dir + "/models/face_detect/face_occlusion_310p.om").c_str()); fd_param.sdk_param.occlusion_modelNames = model_path_occlusion; fd_param.sdk_param.thresld = 0.6; fd_param.sdk_param.devId = m_devId; fd_param.sdk_param.auth_license = "sy_tongtu_aiplatform_sdk_2023"; fd_param.sdk_param.facial_fea_point_config = SY_CONFIG_OPEN; //是否启动关键点检测 fd_param.sdk_param.pose_config = SY_CONFIG_OPEN; //是否启动姿态角 fd_param.sdk_param.quality_config = SY_CONFIG_OPEN; //是否启动质量检测 fd_param.sdk_param.score_config = SY_CONFIG_OPEN; //是否启动人脸置信度 //SY_CONFIG_OPEN SY_CONFIG_CLOSE fd_param.sdk_param.max_result_count = 50; ret = m_face_det_ai_engine.init_ai_engine(fd_param); if (ret < 0 ) { LOG_FATAL("Init face detection failed"); return ret; } #endif // 初始化ws WebsocketClient* pServer = WebsocketClient::getInstance(); if(pServer->init() != 0) { LOG_ERROR("websocket client 初始化失败 !"); return -1; } m_task_param_manager = task_param_manager::getInstance(); m_snapshot_reprocessing = new snapshot_reprocessing(m_devId); m_save_snapshot_reprocessing = new save_snapshot_reprocessing(m_devId); #ifdef POST_USE_RABBITMQ mq_manager_ = new mq::Manager(); #endif VPCUtil* pVpcUtil = VPCUtil::getInstance(); pVpcUtil->init(m_devId); m_pAlgorthimThread = new thread([](void* arg) { CMultiSourceProcess* process = (CMultiSourceProcess*)arg ; process->algorthim_process_thread(); return (void*)0; } , this); m_recode_thread = new thread([](void* arg) { CMultiSourceProcess* process = (CMultiSourceProcess*)arg ; process->recode_thread(); return (void*)0; } , this); m_timing_snapshot_thread = new std::thread( [](void* arg) { CMultiSourceProcess* _this=(CMultiSourceProcess*)arg; if(_this != nullptr){ _this->timing_snapshot_thread(); }else{ LOG_ERROR("定时抓拍线程启动失败 !"); } return (void*)0; }, this); if(nullptr == m_timing_snapshot_thread){ LOG_ERROR("定时抓拍线程启动失败 !"); } LOG_INFO("InitAlgorthim succeed !"); return 0; } #ifdef POST_USE_RABBITMQ /* MQ队列的初始化 */ int CMultiSourceProcess::AddMqConn(mq_type_t mq_type, rabbitmq_conn_params_t mq_conn_param) { /* 初始化MQ队列 */ LOG_INFO("初始化MQ队列"); if(mq_manager_ == nullptr){ LOG_ERROR("mq_manager_ is null"); return FAILED; } if (!mq_manager_->add_conn(mq_type, mq_conn_param)) { LOG_ERROR("Connection MQ failed, ip: {} port: {} uname: {} passwd: {}", mq_conn_param.ip, mq_conn_param.port, mq_conn_param.uname, mq_conn_param.passwd); return MQ_CONN_ERROR; } LOG_INFO("为报警类 绑定MQ回调"); /* 为报警类 绑定回调 传入mq_manager_.publish 内部直接调用*/ if (m_save_snapshot_reprocessing!= nullptr && mq_type_t::ALARM_MQ == mq_type) { m_save_snapshot_reprocessing->set_callback( std::bind(&mq::Manager::publish, mq_manager_, mq_type, std::placeholders::_1, true)); LOG_INFO("为报警类 绑定MQ回调 成功!"); } LOG_INFO("初始化MQ队列成功!"); return SUCCESS; } /* 获取任务的状态 MQ返回 */ int CMultiSourceProcess::GetTaskStatus(const string taskID) { DecoderManager* pDecManager = DecoderManager::getInstance(); std::vector taskids; std::vector statues; if(pDecManager->isPausing(taskID)){ taskids.emplace_back(taskID); statues.emplace_back(2); }else if(pDecManager->isRunning(taskID)){ taskids.emplace_back(taskID); statues.emplace_back(1); } if (!taskids.empty()) { auto json_str = helpers::gen_json::gen_task_status_json(taskids, statues); mq_manager_->publish(mq_type_t::GET_TASK_MQ, json_str.c_str(),true); } return SUCCESS; } #endif bool CMultiSourceProcess::AddTask(task_param _cur_task_param){ DecoderManager* pDecManager = DecoderManager::getInstance(); string task_id = _cur_task_param.task_id; MgrDecConfig config; config.name = task_id; config.cfg.uri = _cur_task_param.ipc_url; config.cfg.post_decoded_cbk = post_decod_cbk; config.cfg.decode_finished_cbk = decode_finished_cbk; config.cfg.force_tcp = true; // rtsp用tcp config.cfg.gpuid = to_string(m_devId); config.cfg.skip_frame = skip_frame_; config.cfg.resize = true; if (1 == _cur_task_param.dec_type){ config.cfg.port = _cur_task_param.port; config.dec_type = DECODER_TYPE_GB28181; if(_cur_task_param.protocal == 0){ // 指定用udp协议 config.cfg.force_tcp = false; } config.cfg.request_stream_cbk = _cur_task_param.gb28181_request_stream_callback ; } else if (2 == _cur_task_param.dec_type){ config.dec_type = DECODER_TYPE_DVPP; } else if (3 == _cur_task_param.dec_type){ config.dec_type = DECODER_TYPE_DVPP_GB28181; if(_cur_task_param.protocal == 0){ // 指定用udp协议 config.cfg.force_tcp = false; } } else { config.dec_type = DECODER_TYPE_FFMPEG; } LOG_INFO("[{}]-create decoder.",task_id); AbstractDecoder* dec = pDecManager->createDecoder(config); if (!dec){ return false; } decode_cbk_userdata* userPtr = new decode_cbk_userdata; userPtr->task_id = task_id; userPtr->opaque = this; userPtr->opaque1 = dec; pDecManager->setPostDecArg(config.name, userPtr); pDecManager->setFinishedDecArg(config.name, userPtr); // pDecManager->setDecKeyframe(config.name, true); // 只对关键帧解码 // 保存新添加任务的配置参数 m_task_param_manager->add_task_param(task_id, _cur_task_param); LOG_INFO("[{}]-start decode.",task_id); // 所有参数都准备好之后再启动解码 bool bStart = pDecManager->startDecodeByName(config.name); if (!bStart){ LOG_INFO("started task {} failed!", config.name); pDecManager->closeDecoderByName(config.name); return false; } #ifdef POST_USE_RABBITMQ mq_type_t mq_type = mq_type_t::ALARM_MQ; pDecManager->set_mq_callback(config.name, std::bind(&mq::Manager::publish, mq_manager_, mq_type, std::placeholders::_1, true)); #endif // 人车物跟踪 if (m_task_param_manager->task_has_vpt_algor(task_id)) vpt_process.addTaskTracker(task_id, 1, 1, skip_frame_); // 人脸跟踪 #ifdef WITH_FACE_DET_SS if (m_task_param_manager->task_has_face_algor(task_id)) m_face_det_ai_engine.add_tracker(task_id, skip_frame_); // 跳帧数暂时写死 #endif m_FinishedTaskMtx.lock(); m_FinishedTaskMap[task_id] = false; m_FinishedTaskMtx.unlock(); int time_interval = m_task_param_manager->get_video_timing_snapshot_interval(task_id); if(time_interval > 0) { dec->setSnapTimeInterval(time_interval); } LOG_INFO("started task {} successed!", config.name); return true; } int CMultiSourceProcess::CountRunningTask() { DecoderManager* pDecManager = DecoderManager::getInstance(); return pDecManager->count(); } void CMultiSourceProcess::decoded_cbk(DeviceMemory* devFrame){ do{ if(m_bfinish){ break; } m_DataListMtx.lock(); if(m_RgbDataList.size() >= 100){ //缓存100帧 m_DataListMtx.unlock(); std::this_thread::sleep_for(std::chrono::milliseconds(3)); continue; } m_RgbDataList.push_back(devFrame); m_DataListMtx.unlock(); break; }while (true); } void CMultiSourceProcess::task_finished(const string task_id){ std::lock_guard l(m_FinishedTaskMtx); m_FinishedTaskMap[task_id] = true; LOG_INFO("task {} finished!", task_id); } bool CMultiSourceProcess::PauseTask(const string taskID){ DecoderManager* pDecManager = DecoderManager::getInstance(); return pDecManager->pauseDecoder(taskID); } bool CMultiSourceProcess::RestartTask(const string taskID){ DecoderManager* pDecManager = DecoderManager::getInstance(); return pDecManager->resumeDecoder(taskID); } bool CMultiSourceProcess::FinishTask(const string taskID){ DecoderManager* pDecManager = DecoderManager::getInstance(); // return pDecManager->closeDecoderByName(taskID); // 记录主动结束的任务 bool flag = pDecManager->closeDecoderByName(taskID); #ifdef POST_USE_RABBITMQ if (flag) { std::lock_guard l1(m_ActiveFinishedTaskMtx); m_ActiveFinishedTaskMap[taskID] = true; } #endif return flag; } int CMultiSourceProcess::snapshot_task(std::string& uri_or_name, const std::string& file_name, bool bInTask) { DecoderManager* pDecManager = DecoderManager::getInstance(); DeviceMemory* devMem = nullptr; if(bInTask) { LOG_INFO("snap in task"); devMem = pDecManager->snapshot_in_task(uri_or_name); } else { LOG_INFO("snap out task"); devMem = pDecManager->snapshot_out_task(uri_or_name, m_devId); } if(devMem != nullptr) { JpegUtil jpegUtil; jpegUtil.jpeg_init(m_devId); acldvppPicDesc *vpcDesc= acldvppCreatePicDesc(); acldvppSetPicDescData(vpcDesc, devMem->getMem()); acldvppSetPicDescFormat(vpcDesc, PIXEL_FORMAT_YUV_SEMIPLANAR_420); acldvppSetPicDescWidth(vpcDesc, devMem->getWidth()); acldvppSetPicDescHeight(vpcDesc, devMem->getHeight()); acldvppSetPicDescWidthStride(vpcDesc, devMem->getWidthStride()); acldvppSetPicDescHeightStride(vpcDesc, devMem->getHeightStride()); acldvppSetPicDescSize(vpcDesc, devMem->getSize()); jpegUtil.jpeg_encode(vpcDesc, file_name); acldvppDestroyPicDesc(vpcDesc); jpegUtil.jpeg_release(); delete devMem; devMem = nullptr; return SUCCESS; } return FAILED; } int CMultiSourceProcess::SnapShot(task_param _cur_task_param) { LOG_INFO("begin SnapShot task: {}", _cur_task_param.task_id); /* 添加任务参数 */ auto &algor_config_param = _cur_task_param.algor_config_params[0]; // 默认算法数量为1 /* 拷贝通用的算法参数 */ auto src_basic_param = algor_config_param.algor_init_config_param->basic_param; if (src_basic_param->result_folder) { CreateResultFolder(src_basic_param->result_folder, ""); } else { src_basic_param->result_folder = ""; } std::string cur_timestamp_ms = std::to_string(helpers::timer::get_timestamp()); std::string file_name = std::string(src_basic_param->result_folder) + helpers::os::sep + _cur_task_param.task_id + "_snap_" + cur_timestamp_ms + ".jpg"; // file_name = "/mnt/data/cmhu/tongtu/Linux_3rdparty/tsl_sdk_20220801/res/snap.jpg"; string task_id = _cur_task_param.task_id; int bSnapShot = snapshot_task(task_id, file_name, true); if (FAILED == bSnapShot){ string url = _cur_task_param.ipc_url; bSnapShot = snapshot_task(url, file_name, false); } if(SUCCESS == bSnapShot){ #ifdef POST_USE_RABBITMQ auto json_str = helpers::gen_json::gen_screen_json(task_id, file_name); // mq_manager_->publish(mq_type_t::GET_TASK_MQ, json_str.c_str(), true); mq_manager_->publish(mq_type_t::SCREENSHORT_TASK_MQ, json_str.c_str(), true); #endif LOG_INFO("SnapShot task: {} success", task_id); } else { LOG_ERROR("SnapShot task: {} failed !", task_id); } return bSnapShot; } void CMultiSourceProcess::CloseAllTask(){ m_bfinish = true; atlas_licence_close(&(skt_handle)); //授权 DecoderManager* pDecManager = DecoderManager::getInstance(); pDecManager->closeAllDecoder(); if(m_pAlgorthimThread){ m_pAlgorthimThread->join(); m_pAlgorthimThread = nullptr; } if(m_recode_thread){ m_recode_thread->join(); m_recode_thread = nullptr; } if(m_timing_snapshot_thread != nullptr && m_timing_snapshot_thread->joinable()){ m_timing_snapshot_thread->join(); m_timing_snapshot_thread = nullptr; } m_DataListMtx.lock(); while (!m_RgbDataList.empty()){ DeviceMemory* gpuMem = m_RgbDataList.front(); delete gpuMem; gpuMem = nullptr; m_RgbDataList.pop_front(); } m_DataListMtx.unlock(); int size = m_RgbDataList.size(); bool bEmpty = m_RgbDataList.empty(); LOG_INFO("CloseAllTask exit."); } void CMultiSourceProcess::CloseAllTask2() { DecoderManager* pDecManager = DecoderManager::getInstance(); pDecManager->closeAllDecoder(); } void CMultiSourceProcess::clear_finished_task(){// 清理已经结束的任务 std::lock_guard l1(m_FinishedTaskMtx); std::lock_guard l2(m_DataListMtx); for (auto iter_finished = m_FinishedTaskMap.begin(); iter_finished!=m_FinishedTaskMap.end(); ){ if(iter_finished->second){ // 解码已经结束 // 判断数据对列中是否还有数据 string task_id = iter_finished->first; bool bFinished = true; for (auto iter = m_RgbDataList.begin(); iter!=m_RgbDataList.end(); ++ iter){ DeviceMemory* gpuMem = *iter; if(task_id == gpuMem->getId()){ bFinished = false; break; } } if (bFinished){ // 解码器已经结束,且数据队列中没有改任务的数据,则做最后任务清理工作 finish_task(task_id,false); iter_finished = m_FinishedTaskMap.erase(iter_finished); pedestrian_vehicle_trespass_.release(task_id); continue; } } ++ iter_finished; } } bool CMultiSourceProcess::finish_task(const string taskID, const bool delete_snapshot){ // 任务结束,关闭跟踪 if (!vpt_process.finishTaskTracker(taskID)) LOG_ERROR("Finish VPT Tracker failed, task_id: {}", taskID); #ifdef WITH_FACE_DET_SS // 人脸任务结束 if (m_task_param_manager->task_has_face_algor(taskID)) m_face_det_ai_engine.finish_task(taskID); #endif #ifdef POST_USE_RABBITMQ { // 外部主动结束的任务不推送mq std::lock_guard mlock(m_ActiveFinishedTaskMtx); if (!(m_ActiveFinishedTaskMap.count(taskID) && m_ActiveFinishedTaskMap[taskID])) { auto json_str = helpers::gen_json::gen_office_task_heart_beat_json({taskID}); mq_manager_->publish(mq_type_t::HEART_BEAT_MQ, json_str.c_str(), true); } if (m_ActiveFinishedTaskMap.count(taskID)) m_ActiveFinishedTaskMap.erase(taskID); } #endif // #ifdef WITH_SECOND_PROCESS pedestrian_vehicle_retrograde_.force_release_result(taskID); //221024 byzsh // #endif #ifdef USE_VILLAGE tricycle_manned_.force_release_result(taskID); truck_manned_.force_release_result(taskID); motor_hsprocess_.force_release_result(taskID); motor_refit_phoneprocess_.force_release_result(taskID); // motor_phoneprocess_.force_release_result(taskID); motor_refitprocess_.force_release_result(taskID); traffic_lightprocess_.force_release_result(taskID); roadseg_corrprocess_.force_release_result(taskID); m_RoadSegTaskMtx.lock(); if (m_RoadSegTaskMap.count(taskID)) { for (int i = 0; i < m_RoadSegTaskMap[taskID].traffic_region.size(); i++) { vector().swap(m_RoadSegTaskMap[taskID].traffic_region[i]); } for (int i = 0; i < m_RoadSegTaskMap[taskID].fence_region.size(); i++) { vector().swap(m_RoadSegTaskMap[taskID].fence_region[i]); } vector().swap(m_RoadSegTaskMap[taskID].labels); vector().swap(m_RoadSegTaskMap[taskID].fence_labels); vector>().swap(m_RoadSegTaskMap[taskID].traffic_region); vector>().swap(m_RoadSegTaskMap[taskID].fence_region); m_RoadSegTaskMap.erase(taskID); } m_RoadSegTaskMtx.unlock(); m_TotalObjMtx.lock(); for(auto ss = m_total_obj_info.begin(); ss != m_total_obj_info.end(); ) { if (strcmp(ss->first.video_id.c_str(), taskID.c_str()) == 0) { vector().swap(ss->second.center_points); m_total_obj_info.erase(ss++); } else ss++; } m_TotalObjMtx.unlock(); #endif m_task_param_manager->delete_task_param(taskID); m_snapshot_reprocessing->release_finished_locus_snapshot(taskID); m_snapshot_reprocessing->release_finished_face_locus_snapshot(taskID); m_snapshot_reprocessing->release_village_finished_locus_snapshot(taskID); return true; } int CMultiSourceProcess::algorthim_process_thread(){ LOG_INFO("algorthim_process_thread start..."); ACL_CALL(aclrtSetDevice(m_devId), ACL_SUCCESS, 1); aclrtContext ctx; ACL_CALL(aclrtCreateContext(&ctx, m_devId), ACL_SUCCESS, 1); while (true){ #ifdef USE_VILLAGE if (!CheckTime()) { //时间限制 LOG_FATAL("authority failed!"); break; } #else if (check_label == -1) { //机器授权 LOG_FATAL("authority failed!"); break; } if (!UpdateLabel()) { LOG_FATAL("authority failed!"); break; } #endif if(m_bfinish){ break; } clear_finished_task(); vector vec_gpuMem; m_DataListMtx.lock(); while (!m_RgbDataList.empty()){ DeviceMemory* gpuMem = m_RgbDataList.front(); if(gpuMem->getMem() == nullptr) { LOG_WARN("mem is null"); } else { vec_gpuMem.push_back(gpuMem); string task_id = gpuMem->getId(); if (!pedestrian_vehicle_trespass_.isInited(task_id)) { auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map != nullptr) { int input_image_width = gpuMem->getWidth(); int input_image_height = gpuMem->getHeight(); // LOG_DEBUG("task_id: {} width: {} height:{}", task_id, input_image_width, input_image_height); /* 如果开启了行人 机动车非法闯入功能 生成闯入区域mask */ if (algor_map->find(algorithm_type_t::PEDESTRIAN_TRESPASS) != algor_map->end()) { pedestrian_vehicle_trespass_.pedestrianvehicletrespass_init_region( task_id, algorithm_type_t::PEDESTRIAN_TRESPASS, input_image_width, input_image_height); } if (algor_map->find(algorithm_type_t::VEHICLE_TRESPASS) != algor_map->end()) { pedestrian_vehicle_trespass_.pedestrianvehicletrespass_init_region( task_id, algorithm_type_t::VEHICLE_TRESPASS, input_image_width, input_image_height); } } } } m_RgbDataList.pop_front(); if(vec_gpuMem.size() >= m_batch_size){ break; } } m_DataListMtx.unlock(); if(vec_gpuMem.size() <= 0){ std::this_thread::sleep_for(std::chrono::milliseconds(3)); continue; } aclrtSetCurrentContext(ctx); // auto record_t1 = std::chrono::system_clock::now(); #ifdef USE_VILLAGE algorthim_road_seg(vec_gpuMem); #endif auto vptprocess1 = std::chrono::system_clock::now(); algorthim_vpt(vec_gpuMem); auto vptprocess2 = std::chrono::system_clock::now(); auto vptprocess_time = std::chrono::duration_cast(vptprocess2 - vptprocess1).count(); // LOG_INFO("Process per frame cost time: {} ms fps: {}", vptprocess_time/vec_gpuMem.size(), 1000/(vptprocess_time/vec_gpuMem.size())); #ifdef WITH_FACE_DET_SS algorthim_face_detect(vec_gpuMem); #endif // static int index = 0; // index++; // DeviceMemory* mem = vec_gpuMem[0]; // if(index % 100 == 0){ // string task_id = mem->getId(); // string video_file_name = "/home/cmhu/vpt_ascend_arm/bin/res/video_recode/"; // video_file_name += task_id + "_" + std::to_string(mem->getFrameNb()) + ".mp4"; // RecoderInfo recoderInfo; // recoderInfo.task_id = task_id; // recoderInfo.object_id = std::to_string(index); // recoderInfo.recoderPath = video_file_name; // recoderInfo.frame_nb = mem->getFrameNb(); // recoderInfo.mq_info = task_id + "_" + std::to_string(mem->getFrameNb()); // m_recoderinfo_queue_mtx.lock(); // m_recoderinfo_queue.push_back(recoderInfo); // m_recoderinfo_queue_mtx.unlock(); // } // auto record_t2 = std::chrono::system_clock::now(); // auto record_time = std::chrono::duration_cast(record_t2 - record_t1).count(); // LOG_INFO("Record per frame cost time: {} ms fps: {}", record_time/vec_gpuMem.size(), 1000/(record_time/vec_gpuMem.size())); for(int i=0;i < vec_gpuMem.size(); i++){ DeviceMemory* mem = vec_gpuMem[i]; if(mem->getSize() <= 0){ continue; } delete mem; mem = nullptr; } vec_gpuMem.clear(); vector().swap(vec_gpuMem); // free. } aclrtDestroyContext(ctx); aclrtResetDevice(m_devId); LOG_INFO("algorthim_process_thread exit."); return 0; } int CMultiSourceProcess::algorthim_road_seg(vector vec_gpuMem) { vector roadSeg_task_id; vector roadSeg_imgs; vector vec_segMem; m_RoadSegTaskMtx.lock(); for (int i = 0; i < vec_gpuMem.size(); i++) { DeviceMemory* mem = vec_gpuMem[i]; long long frame_number = mem->getFrameNb(); if (m_RoadSegTaskMap[mem->getId()].isseg && frame_number % 15000) { //10min重复一次 continue; } LOG_INFO("task: {} frame_num: {} perform road_seg.", mem->getId(), mem->getFrameNb()); sy_img img; img.w_ = mem->getWidth(); img.h_ = mem->getHeight(); img.c_ = mem->getChannel(); img.data_ = mem->getMem(); roadSeg_imgs.push_back(img); roadSeg_task_id.push_back(mem->getId()); vec_segMem.push_back(mem); m_RoadSegTaskMap[mem->getId()].isseg = true; // } m_RoadSegTaskMtx.unlock(); /* 待检测的图片不为空 开始检测 */ if (!roadSeg_imgs.empty()) { vector>> traffic_region, fence_region; //道路区域(多个) vector> labels, fence_labels; //道路区域对应的类别:机动车道、非机动车道等 seg_process.process_gpu(roadSeg_imgs.data(), vec_segMem, roadSeg_task_id, traffic_region, labels); seg3cls_process.process_gpu(roadSeg_imgs.data(), vec_segMem, roadSeg_task_id, fence_region, fence_labels); int bidx = 0; m_RoadSegTaskMtx.lock(); for (auto _task_id_iter = roadSeg_task_id.begin(); _task_id_iter != roadSeg_task_id.end(); ++_task_id_iter, ++ bidx) { auto task_id = *_task_id_iter; m_RoadSegTaskMap[task_id].traffic_region.assign(traffic_region[bidx].begin(), traffic_region[bidx].end()); m_RoadSegTaskMap[task_id].labels.assign(labels[bidx].begin(), labels[bidx].end()); m_RoadSegTaskMap[task_id].fence_region.assign(fence_region[bidx].begin(), fence_region[bidx].end()); m_RoadSegTaskMap[task_id].fence_labels.assign(fence_labels[bidx].begin(), fence_labels[bidx].end()); m_RoadSegTaskMap[task_id].isseg = true; } m_RoadSegTaskMtx.unlock(); } vec_segMem.clear(); roadSeg_task_id.clear(); roadSeg_imgs.clear(); return 0; } int CMultiSourceProcess::algorthim_vpt(vector vec_gpuMem){ vector vpt_interest_task_id; vector vpt_interest_imgs; vector vec_vptMem; for (int i = 0; i < vec_gpuMem.size(); i++) { DeviceMemory* mem = vec_gpuMem[i]; if (!m_task_param_manager->task_has_vpt_algor(mem->getId())){ continue; } sy_img img; img.w_ = mem->getWidth(); img.h_ = mem->getHeight(); img.c_ = mem->getChannel(); img.data_ = mem->getMem(); vpt_interest_imgs.push_back(img); vpt_interest_task_id.push_back(mem->getId()); vec_vptMem.push_back(mem); } /* 待检测的图片不为空 开始检测 */ if (!vpt_interest_imgs.empty()) { vector> deleteObjectID; deleteObjectID.resize(vpt_interest_task_id.size()); vector> unUsedResult; vector vptResult(0); /* 一级检测器,内部已完成跟踪操作 */ vpt_process.process_gpu(vpt_interest_imgs.data(), vpt_interest_task_id, vptResult, deleteObjectID, unUsedResult); // do det & track. // vpt_process.process_gpu(vpt_interest_imgs.data(), vec_vptMem, vpt_interest_task_id, vptResult, deleteObjectID, unUsedResult); // debug m_snapshot_reprocessing->screen_effective_snapshot(vptResult); #ifndef VEHICLE_MULTI_BOXES /* 快照优选(内部可实现不同的快照优选策略) */ m_snapshot_reprocessing->update_bestsnapshot(vec_vptMem, vptResult, deleteObjectID); /* for snapshot algorithm. 轨迹结束目标 做最后的结果返回(当前返回算法结果+快照保存路径)*/ vehicle_snapshot(vpt_interest_task_id, deleteObjectID); #else algorithm_vehicle_relult(vec_vptMem, vptResult, deleteObjectID); send_locus_finished_msg(vpt_interest_task_id, deleteObjectID); #endif // #ifdef WITH_SECOND_PROCESS /* for pedestrian safety det. 行人安全分析算法模块 */ // algorthim_pedestrian_safety(vpt_interest_task_id, vpt_interest_imgs,vptResult); // 逆行 algorthim_retrograde(vpt_interest_task_id, vec_vptMem, vptResult); retrograde_snapshot(vpt_interest_task_id, deleteObjectID); // 闯入 algorthim_trespass(vpt_interest_task_id, vec_vptMem, vptResult, deleteObjectID); trespass_snapshot(vpt_interest_task_id, deleteObjectID); // #endif #ifdef USE_VILLAGE // 轨迹记录 trace_record(vpt_interest_task_id, vptResult); // 农村违法分析的快照缓存 m_snapshot_reprocessing->update_village_bestsnapshot(vec_vptMem, vptResult, deleteObjectID); // 三轮车载人 algorithm_tricycle_manned(vpt_interest_task_id, vec_vptMem, vptResult); // 货车载人 algorithm_truck_manned(vpt_interest_task_id, vec_vptMem, vptResult); // 二轮车超员/未戴盔 algorithm_motor_hs_process(vpt_interest_task_id, vec_vptMem, vptResult); // 二轮车驾乘人员使用手机(人骑车辅助版本) algorithm_motor_refit_phone_process(vpt_interest_task_id, vec_vptMem, vptResult); /*// 二轮车驾乘人员使用手机 algorithm_motor_phone_process(vpt_interest_task_id, vec_vptMem, vptResult);*/ // 电动车改装(加雨棚) algorithm_motor_refit_process(vpt_interest_task_id, vec_vptMem, vptResult); // 行人/二轮车闯红灯 algorithm_traffic_light_process(vpt_interest_task_id, vec_vptMem, vptResult); // 与道路分割相关的算法 algorithm_roadseg_correlation_process(vpt_interest_task_id, vec_vptMem, vptResult); village_snapshot(vpt_interest_task_id, vec_vptMem, deleteObjectID); #endif vptResult.clear(); unUsedResult.clear(); // deleteObjectID.clear(); for (int i = 0; i < deleteObjectID.size(); i++) { vector().swap(deleteObjectID[i]); // free. } vector>().swap(deleteObjectID); // free. } vec_vptMem.clear(); vpt_interest_task_id.clear(); vpt_interest_imgs.clear(); return 0; } int CMultiSourceProcess::algorithm_vehicle_relult(vector vec_devMem, vector& vptResult, vector>& delete_object_id) { vector results = m_snapshot_reprocessing->get_vehicle_snapshot(vec_devMem, vptResult, skip_frame_); VPCUtil* pVPCUtil = VPCUtil::getInstance(); for (auto &result : results) { if(result.objs.size() <= 0){ continue; } auto task_id = result.task_id; auto task_other_params = m_task_param_manager->get_task_other_param(task_id); const auto &algor_other_params = task_other_params->find(algorithm_type_t::VEHICLE_SNAPSHOT); if (algor_other_params == task_other_params->end()) { LOG_ERROR("taskId {} not found algor {}", task_id.c_str(), (int)algorithm_type_t::VEHICLE_SNAPSHOT); continue; } vector vec_obj_info_list = pVPCUtil->crop_batch(result.memPtr, result.objs); if(vec_obj_info_list.size() != result.objs.size()){ LOG_ERROR("vpc_crop size error !"); VPCUtil::vpc_imgList_release(vec_obj_info_list); continue; } const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; std::string cur_timestamp_ms = std::to_string(helpers::timer::get_cur_time_ms()); const std::string fpath_origin = basic_param->result_folder + helpers::os::sep + task_id + "_" + std::to_string(result.objs.size()) + "_" + std::to_string(result.id) + "_" + cur_timestamp_ms + ".jpg"; ImgSaveInfo saveInfo; saveInfo.file_path = fpath_origin; saveInfo.img_info = VPCUtil::vpc_devMem2vpcImg(result.memPtr); m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(saveInfo); // 保存抠图并发MQ for(int i =0; i < result.objs.size(); i++){ video_object_info obj = result.objs[i]; std::string cur_timestamp_ms = std::to_string(helpers::timer::get_cur_time_ms()); const std::string fpath_roi = basic_param->result_folder_little + helpers::os::sep + task_id + "_" + std::to_string(obj.object_id) + "_" + cur_timestamp_ms + ".jpg"; video_object_snapshot new_obj_ss_info; new_obj_ss_info.analysisRes = nullptr; new_obj_ss_info.object_id = obj.object_id; new_obj_ss_info.obj_info.set_data(obj.index, obj.confidence, obj.left, obj.top, obj.right, obj.bottom); strcpy(new_obj_ss_info.task_id, task_id.c_str()); strcpy(new_obj_ss_info.video_image_path, fpath_origin.c_str()); strcpy(new_obj_ss_info.snapshot_image_path, fpath_roi.c_str()); new_obj_ss_info.nFinished = 0; string json_str = helpers::gen_json::gen_multi_obj_json(algorithm_type_t::VEHICLE_SNAPSHOT, new_obj_ss_info); ImgSaveInfo save_info; save_info.file_path = fpath_roi; save_info.img_info = vec_obj_info_list[i]; save_info.json_str = json_str; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(save_info); OBJ_KEY obj_key{task_id, obj.object_id}; // 保存有轨迹的object记录,轨迹结束时需要发消息 std::lock_guard l(m_total_mutex); m_total_snapshot_info_multi_object.insert(obj_key); } vec_obj_info_list.clear(); } return 0; } void CMultiSourceProcess::send_locus_finished_msg(vector& vpt_interest_task_id, vector> deleteObjectID){ auto task_iter = vpt_interest_task_id.begin(); for (int i = 0; i < deleteObjectID.size(); i++, ++task_iter) // loop taskId. { string task_id = *task_iter; for (int &j : deleteObjectID[i]) // loop algor type. { OBJ_KEY obj_key = {task_id, j}; auto task_param_ptr = m_task_param_manager->get_task_algor_param(task_id); // 该路任务开启了抓拍功能 开始抓拍保存;若未开启抓拍,清空显存资源 if (task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_SNAPSHOT) != task_param_ptr->vehicle_algors.end()) { std::lock_guard l(m_total_mutex); if (m_total_snapshot_info_multi_object.find(obj_key) != m_total_snapshot_info_multi_object.end()) { video_object_snapshot new_obj_ss_info; new_obj_ss_info.object_id = j; new_obj_ss_info.nFinished = 1; strcpy(new_obj_ss_info.task_id, task_id.c_str()); string json_str = helpers::gen_json::gen_multi_obj_json(algorithm_type_t::VEHICLE_SNAPSHOT, new_obj_ss_info); // 通知结束的轨迹 ImgSaveInfo save_info; save_info.json_str = json_str; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(save_info); m_total_snapshot_info_multi_object.erase(obj_key); } } } } } // for snapshot algorithm. 轨迹结束目标 做最后的结果返回(当前返回算法结果+快照保存路径) void CMultiSourceProcess::vehicle_snapshot(vector& vpt_interest_task_id, vector> deleteObjectID) { auto task_iter = vpt_interest_task_id.begin(); for (int i = 0; i < deleteObjectID.size(); i++, ++task_iter) // loop taskId. { for (int &j : deleteObjectID[i]) // loop algor type. { OBJ_KEY obj_key = {*task_iter, j}; // 该路任务开启了抓拍功能 开始抓拍保存;若未开启抓拍,清空显存资源 vehicle_locus_finished(obj_key); } } } void CMultiSourceProcess::vehicle_locus_finished(const OBJ_KEY obj_key) { auto task_param_ptr = m_task_param_manager->get_task_algor_param(obj_key.video_id); if (task_param_ptr == nullptr){ return; } // 该路任务开启了抓拍功能 开始抓拍保存;若未开启抓拍,清空显存资源 if ((task_param_ptr->human_algors.find(algorithm_type_t::HUMAN_SNAPSHOT) == task_param_ptr->human_algors.end() && task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_SNAPSHOT) != task_param_ptr->vehicle_algors.end() && task_param_ptr->nonmotor_vehicle_algors.find(algorithm_type_t::NONMOTOR_VEHICLE_SNAPSHOT) != task_param_ptr->nonmotor_vehicle_algors.end())) { return; } map _total_snapshot_info = m_snapshot_reprocessing->get_total_snapshot_info(); if(_total_snapshot_info.size() <= 0){ return; } LOG_DEBUG("_total_snapshot_info size: {}", _total_snapshot_info.size()); auto it = _total_snapshot_info.find(obj_key); if (it == _total_snapshot_info.end()) { return; } const OBJ_VALUE obj_value = it->second; auto task_other_param_map = m_task_param_manager->get_task_other_params(); auto it_other_param = task_other_param_map.find(obj_key.video_id); if(it_other_param == task_other_param_map.end()){ return; } auto task_other_param = it_other_param->second; // LOG_DEBUG("algor_type"); auto algor_type = index_to_algo_type[obj_value.index.index]; // LOG_DEBUG("task_other_param"); auto it_algor_type = task_other_param.find(algor_type); if(it_algor_type == task_other_param.end()) { return; } // LOG_DEBUG("algor_param"); auto algor_param = (algor_config_param_snapshot *)it_algor_type->second->algor_param; if(algor_param == nullptr) { return; } if (obj_value.confidence <= algor_param->threshold) { LOG_DEBUG("Snapshot conf filter ({} vs {})", obj_value.confidence, algor_param->threshold); // 删除记录,同时释放显存 m_snapshot_reprocessing->release_finished_locus_snapshot(obj_key.video_id, obj_key.obj_id, true); return; } // LOG_DEBUG("cur_param"); const algor_basic_config_param_t *cur_param = it_algor_type->second->basic_param; // const algor_basic_config_param_t *cur_param = task_other_param[algor_type]->basic_param; if(cur_param == nullptr) { return; } // 原图 // LOG_DEBUG("原图"); std::string cur_timestamp_ms = std::to_string(helpers::timer::get_cur_time_ms()); std::string fpath_origin = std::string(cur_param->result_folder) + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + cur_timestamp_ms + ".jpg"; ImgSaveInfo origin_save_info; origin_save_info.file_path = fpath_origin; origin_save_info.img_info = obj_value.snapShot; origin_save_info.obj_rect = obj_value.obj_pos; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(origin_save_info); { // LOG_DEBUG("抠图"); // 抠图 string json_str = ""; string object_file_name = std::string(cur_param->result_folder_little) + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + cur_timestamp_ms + ".jpg"; #ifdef POST_USE_RABBITMQ video_object_snapshot new_obj_ss_info; new_obj_ss_info.analysisRes = nullptr; new_obj_ss_info.object_id = obj_key.obj_id; new_obj_ss_info.obj_info.set_data(obj_value.index.index, obj_value.confidence, obj_value.obj_pos.left_, obj_value.obj_pos.top_, obj_value.obj_pos.left_ + obj_value.obj_pos.width_, obj_value.obj_pos.top_ + obj_value.obj_pos.height_); strcpy(new_obj_ss_info.task_id, obj_key.video_id.c_str()); strcpy(new_obj_ss_info.video_image_path, fpath_origin.c_str()); strcpy(new_obj_ss_info.snapshot_image_path, object_file_name.c_str()); json_str = helpers::gen_json::gen_snapshot_json(algor_type, new_obj_ss_info); #endif ImgSaveInfo obj_save_info; obj_save_info.file_path = object_file_name; obj_save_info.img_info = obj_value.snapShotLittle; obj_save_info.json_str = json_str; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(obj_save_info); } LOG_DEBUG("仅删除记录,显存会在保存线程中删除"); // 仅删除记录,显存会在保存线程中删除 m_snapshot_reprocessing->release_finished_locus_snapshot(obj_key.video_id, obj_key.obj_id, false); } void CMultiSourceProcess::timing_snapshot_thread(){ LOG_INFO("timing_snapshot_thread start."); JpegUtil jpegUtil; jpegUtil.jpeg_init(m_devId); DecoderManager* pDecManager = DecoderManager::getInstance(); while(!m_bfinish){ vector vec_devMem = pDecManager->timing_snapshot_all(); for (auto devMem : vec_devMem) { auto task_id = devMem->getId(); auto task_other_params = m_task_param_manager->get_task_other_param(task_id); if (task_other_params == nullptr) continue; const auto &algor_other_params = task_other_params->find(algorithm_type_t::VIDEO_TIMING_SNAPSHOT); if(algor_other_params != task_other_params->end()){ const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; std::string cur_timestamp_ms = std::to_string(devMem->getTimesstamp()); const std::string fpath_ori = basic_param->result_folder + helpers::os::sep + task_id + "_" + cur_timestamp_ms + ".jpg"; // LOG_DEBUG("snapshot save path: {}", fpath_ori.c_str()); acldvppPicDesc *vpcDesc= acldvppCreatePicDesc(); acldvppSetPicDescData(vpcDesc, devMem->getMem()); acldvppSetPicDescFormat(vpcDesc, PIXEL_FORMAT_YUV_SEMIPLANAR_420); acldvppSetPicDescWidth(vpcDesc, devMem->getWidth()); acldvppSetPicDescHeight(vpcDesc, devMem->getHeight()); acldvppSetPicDescWidthStride(vpcDesc, devMem->getWidthStride()); acldvppSetPicDescHeightStride(vpcDesc, devMem->getHeightStride()); acldvppSetPicDescSize(vpcDesc, devMem->getSize()); jpegUtil.jpeg_encode(vpcDesc, fpath_ori); acldvppDestroyPicDesc(vpcDesc); vpcDesc = nullptr; #ifdef POST_USE_RABBITMQ auto json_str = helpers::gen_json::gen_vtsnapshot_json(task_id, fpath_ori); mq_manager_->publish(mq_type_t::TIMING_SCREENSHORT_TASK_MQ, json_str.c_str(), true); #endif } } for(auto devMem : vec_devMem){ delete devMem; devMem = nullptr; } vec_devMem.clear(); std::this_thread::sleep_for(std::chrono::milliseconds(600)); } jpegUtil.jpeg_release(); LOG_INFO("timing_snapshot_thread end."); } // 逆行 void CMultiSourceProcess::algorthim_retrograde(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult){ vector interest_task_id; vector interest_vpt_result; vector interest_imgs; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::PEDESTRIAN_RETROGRADE) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_RETROGRADE) != algor_map->end()) { interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } // LOG_DEBUG("retrograde interest_vpt_result size: {}", interest_vpt_result.size()); if (!interest_imgs.empty()){ pedestrian_vehicle_retrograde_.update_mstreams(interest_task_id, interest_imgs, interest_vpt_result); } } // 闯入 void CMultiSourceProcess::algorthim_trespass(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult ,vector>& deleteObjectID){ vector trespass_interest_task_id; vector trespass_interest_vpt_result; vector> trespass_interest_deleteobjs; vector trespass_interest_imgs; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::PEDESTRIAN_TRESPASS) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_TRESPASS) != algor_map->end()) { trespass_interest_task_id.emplace_back(task_id); trespass_interest_imgs.emplace_back(vpt_interest_imgs[_idx]); trespass_interest_vpt_result.emplace_back(vptResult[_idx]); trespass_interest_deleteobjs.emplace_back(deleteObjectID[_idx]); } } // LOG_DEBUG("trespass interest_vpt_result size: {}", trespass_interest_vpt_result.size()); if (!trespass_interest_imgs.empty()) { pedestrian_vehicle_trespass_.update_mstreams( trespass_interest_task_id, trespass_interest_imgs, trespass_interest_vpt_result, trespass_interest_deleteobjs); } } // 轨迹记录 void CMultiSourceProcess::trace_record(vector& vpt_interest_task_id, vector& vptResult) { int bidx = 0; for (auto iter = vpt_interest_task_id.begin(); iter != vpt_interest_task_id.end(); ++ iter, ++ bidx) { auto task_id = *iter; for (int c = 0; c < vptResult[bidx].obj_count; c++) { //计算检测框坐标中心点 sy_point center_point; center_point.x_ = vptResult[bidx].obj[c].left + (vptResult[bidx].obj[c].right - vptResult[bidx].obj[c].left) * 0.5; // center_point.y_ = vptResult[bidx].obj[c].top + (vptResult[bidx].obj[c].bottom - vptResult[bidx].obj[c].top) * 0.85; center_point.y_ = vptResult[bidx].obj[c].bottom; OBJ_KEY new_obj = { task_id, vptResult[bidx].obj[c].id }; m_TotalObjMtx.lock(); m_total_obj_info[new_obj].index = vptResult[bidx].obj[c].index; m_total_obj_info[new_obj].center_points.push_back(center_point); m_TotalObjMtx.unlock(); } } } // 三轮车载人 void CMultiSourceProcess::algorithm_tricycle_manned(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult) { vector interest_task_id; vector interest_vpt_result; vector interest_imgs; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::TRICYCLE_MANNED) != algor_map->end()) { interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } // LOG_DEBUG("tricycle_manned interest_vpt_result size: {}", interest_vpt_result.size()); if (!interest_imgs.empty()){ tricycle_manned_.update_mstreams(interest_task_id, interest_imgs, interest_vpt_result); } } // 货车载人 void CMultiSourceProcess::algorithm_truck_manned(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult) { vector interest_task_id; vector interest_vpt_result; vector interest_imgs; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::TRUCK_MANNED) != algor_map->end()) { interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } // LOG_DEBUG("truck_manned interest_vpt_result size: {}", interest_vpt_result.size()); if (!interest_imgs.empty()){ m_TotalObjMtx.lock(); truck_manned_.update_mstreams(interest_task_id, interest_imgs, m_total_obj_info, interest_vpt_result); m_TotalObjMtx.unlock(); } } // 二轮车超员/未戴盔 void CMultiSourceProcess::algorithm_motor_hs_process(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult) { vector interest_task_id; vector interest_vpt_result; vector interest_imgs; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::NONMOTOR_VEHICLE_NOHELMET) != algor_map->end() || algor_map->find(algorithm_type_t::NONMOTOR_VEHICLE_OVERMAN) != algor_map->end()) { interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } // LOG_DEBUG("motor_hsprocess interest_vpt_result size: {}", interest_vpt_result.size()); if (!interest_imgs.empty()){ motor_hsprocess_.update_mstreams(interest_task_id, interest_imgs, interest_vpt_result); } } // 二轮车驾乘人员使用手机(人骑车辅助版) void CMultiSourceProcess::algorithm_motor_refit_phone_process(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult) { vector interest_task_id; vector interest_vpt_result; vector interest_imgs; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); // if (algor_map->find(algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE) != algor_map->end() || // algor_map->find(algorithm_type_t::NONMOTOR_VEHICLE_REFIT) != algor_map->end()) { if (algor_map->find(algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE) != algor_map->end()) { interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } if (!interest_imgs.empty()){ motor_refit_phoneprocess_.update_mstreams(interest_task_id, interest_imgs, interest_vpt_result); } } /*// 二轮车驾乘人员使用手机 void CMultiSourceProcess::algorithm_motor_phone_process(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult) { vector interest_task_id; vector interest_vpt_result; vector interest_imgs; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE) != algor_map->end()) { interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } if (!interest_imgs.empty()){ motor_phoneprocess_.update_mstreams(interest_task_id, interest_imgs, interest_vpt_result); } }*/ // 电动车改装(加雨棚) void CMultiSourceProcess::algorithm_motor_refit_process(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult) { vector interest_task_id; vector interest_vpt_result; vector interest_imgs; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::NONMOTOR_VEHICLE_REFIT) != algor_map->end()) { interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } if (!interest_imgs.empty()){ motor_refitprocess_.update_mstreams(interest_task_id, interest_imgs, interest_vpt_result); } } // 行人/二轮车闯红灯 void CMultiSourceProcess::algorithm_traffic_light_process(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult) { vector interest_task_id; vector interest_vpt_result; vector interest_imgs; vector sy_interest_imgs; vector>> interest_traffic_region; vector> interest_labels; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::PERSON_RUNNING_REDLIGHTS) != algor_map->end() || algor_map->find(algorithm_type_t::NONMOTOR_RUNNING_REDLIGHTS) != algor_map->end()) { m_RoadSegTaskMtx.lock(); if (!m_RoadSegTaskMap[task_id].isseg) continue; else { interest_traffic_region.push_back(m_RoadSegTaskMap[task_id].traffic_region); interest_labels.push_back(m_RoadSegTaskMap[task_id].labels); } m_RoadSegTaskMtx.unlock(); DeviceMemory* mem = vpt_interest_imgs[_idx]; sy_img img; img.w_ = mem->getWidth(); img.h_ = mem->getHeight(); img.c_ = mem->getChannel(); img.data_ = mem->getMem(); sy_interest_imgs.push_back(img); interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } if (!interest_imgs.empty()){ m_TotalObjMtx.lock(); traffic_lightprocess_.update_mstreams(interest_task_id, sy_interest_imgs, interest_imgs, interest_traffic_region, interest_labels, m_total_obj_info, interest_vpt_result); m_TotalObjMtx.unlock(); } } // 与道路分割相关的算法 void CMultiSourceProcess::algorithm_roadseg_correlation_process(vector& vpt_interest_task_id, vector vpt_interest_imgs, vector& vptResult) { vector interest_task_id; vector interest_vpt_result; vector interest_imgs; vector sy_interest_imgs; vector>> interest_traffic_region, interest_fence_region; vector> interest_labels, interest_fence_labels; int _idx = 0; for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); ++_task_id_iter, ++_idx) // loop task_id; { auto task_id = *_task_id_iter; auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::PERSON_IN_VEHICLELANE) != algor_map->end() || algor_map->find(algorithm_type_t::NONMOTOR_IN_VEHICLELANE) != algor_map->end() || algor_map->find(algorithm_type_t::NONMOTOR_CEOSSPARKLINE) != algor_map->end() || algor_map->find(algorithm_type_t::PERSON_CROSS) != algor_map->end() || algor_map->find(algorithm_type_t::NONMOTOR_WRONGDIRECTION) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_WRONGDIRECTION) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_NOTGIVEWAY) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_SOLIDLINETURNAROUND) != algor_map->end() || algor_map->find(algorithm_type_t::VEHICLE_NOTDECELERATION) != algor_map->end() ) { m_RoadSegTaskMtx.lock(); if (!m_RoadSegTaskMap[task_id].isseg) continue; else { interest_traffic_region.push_back(m_RoadSegTaskMap[task_id].traffic_region); interest_labels.push_back(m_RoadSegTaskMap[task_id].labels); interest_fence_region.push_back(m_RoadSegTaskMap[task_id].fence_region); interest_fence_labels.push_back(m_RoadSegTaskMap[task_id].fence_labels); } m_RoadSegTaskMtx.unlock(); DeviceMemory* mem = vpt_interest_imgs[_idx]; sy_img img; img.w_ = mem->getWidth(); img.h_ = mem->getHeight(); img.c_ = mem->getChannel(); img.data_ = mem->getMem(); sy_interest_imgs.push_back(img); interest_task_id.emplace_back(task_id); interest_imgs.emplace_back(vpt_interest_imgs[_idx]); interest_vpt_result.emplace_back(vptResult[_idx]); } } if (!interest_imgs.empty()){ m_TotalObjMtx.lock(); roadseg_corrprocess_.update_mstreams(interest_task_id, sy_interest_imgs, interest_imgs, interest_traffic_region, interest_labels, interest_fence_region, interest_fence_labels, m_total_obj_info, interest_vpt_result); m_TotalObjMtx.unlock(); } } // for snapshot algorithm. 轨迹结束目标 做最后的结果返回(当前返回算法结果+快照保存路径) void CMultiSourceProcess::village_snapshot(vector& vpt_interest_task_id, vector vec_vptMem, vector> deleteObjectID) { auto task_iter = vpt_interest_task_id.begin(); for (int i = 0; i < deleteObjectID.size(); i++, ++task_iter) // loop taskId. { auto task_id = *task_iter; for (int &j : deleteObjectID[i]) // loop algor type. { OBJ_KEY obj_key = {*task_iter, j}; auto task_param_ptr = m_task_param_manager->get_task_algor_param(obj_key.video_id); auto algor_config_param = m_task_param_manager->get_task_algor_params(); auto algor_param = m_task_param_manager->get_task_other_params(); if (!(algor_config_param.count(task_id) && algor_param.count(task_id))) continue; bool village_alarm = false; std::string video_folder = "", result_folder = "", result_folder_little = ""; //要求各事件使用同一个视频保存地址,否则会被最后一个事件的地址覆盖 auto task_other_params = m_task_param_manager->get_task_other_param(task_id); std::vector algorithm_types; bool save_single_algor_pic = false; // 是否保存单算法的报警图片(若设置为true需先去掉算法内部注释!!!) if (task_param_ptr->nonmotor_vehicle_algors.find(algorithm_type_t::TRICYCLE_MANNED) != task_param_ptr->nonmotor_vehicle_algors.end()) { const auto &algor_other_params = task_other_params->find(algorithm_type_t::TRICYCLE_MANNED); const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; auto result = tricycle_manned_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algorithm_type_t::TRICYCLE_MANNED}); if (result.get()) { village_alarm = true; algorithm_types.push_back((int)algorithm_type_t::TRICYCLE_MANNED); video_folder = basic_param->video_folder; result_folder = basic_param->result_folder; result_folder_little = basic_param->result_folder_little; // 原图及抠图 vpc_img_info src_img; src_img.pic_desc = result->origin_img_desc; src_img.task_id = obj_key.video_id; vpc_img_info roi_img; roi_img.pic_desc = result->roi_img_desc; roi_img.task_id = obj_key.video_id; roi_img.object_id = obj_key.obj_id; if (save_single_algor_pic) { auto json_str = helpers::gen_json::gen_manned_json(obj_key.video_id, obj_key.obj_id, result->box, algorithm_type_t::TRICYCLE_MANNED); save_snapshot_process(obj_key, algorithm_type_t::TRICYCLE_MANNED, src_img, roi_img, 0, json_str); } else { VPCUtil::vpc_img_release(src_img); VPCUtil::vpc_img_release(roi_img); } } } if (task_param_ptr->vehicle_algors.find(algorithm_type_t::TRUCK_MANNED) != task_param_ptr->vehicle_algors.end()) { const auto &algor_other_params = task_other_params->find(algorithm_type_t::TRUCK_MANNED); const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; auto result = truck_manned_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algorithm_type_t::TRUCK_MANNED}); if (result.get()) { village_alarm = true; algorithm_types.push_back((int)algorithm_type_t::TRUCK_MANNED); video_folder = basic_param->video_folder; result_folder = basic_param->result_folder; result_folder_little = basic_param->result_folder_little; // 原图及抠图 vpc_img_info src_img; src_img.pic_desc = result->origin_img_desc; src_img.task_id = obj_key.video_id; vpc_img_info roi_img; roi_img.pic_desc = result->roi_img_desc; roi_img.task_id = obj_key.video_id; roi_img.object_id = obj_key.obj_id; if (save_single_algor_pic) { auto json_str = helpers::gen_json::gen_manned_json(obj_key.video_id, obj_key.obj_id, result->box, algorithm_type_t::TRUCK_MANNED); save_snapshot_process(obj_key, algorithm_type_t::TRUCK_MANNED, src_img, roi_img, 0, json_str); } else { VPCUtil::vpc_img_release(src_img); VPCUtil::vpc_img_release(roi_img); } } } vector motorhs_algor = {algorithm_type_t::NONMOTOR_VEHICLE_NOHELMET, algorithm_type_t::NONMOTOR_VEHICLE_OVERMAN}; for (size_t idx = 0; idx < motorhs_algor.size(); ++idx) { if (algor_param[task_id].count(motorhs_algor.at(idx))) { const auto &algor_other_params = task_other_params->find(motorhs_algor.at(idx)); const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; auto result = motor_hsprocess_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, motorhs_algor.at(idx)}); if (result.get()) { village_alarm = true; algorithm_types.push_back((int)motorhs_algor.at(idx)); video_folder = basic_param->video_folder; result_folder = basic_param->result_folder; result_folder_little = basic_param->result_folder_little; // 原图及抠图 vpc_img_info src_img; src_img.pic_desc = result->origin_img_desc; src_img.task_id = obj_key.video_id; vpc_img_info roi_img; roi_img.pic_desc = result->roi_img_desc; roi_img.task_id = obj_key.video_id; roi_img.object_id = obj_key.obj_id; if (save_single_algor_pic) { auto json_str = helpers::gen_json::gen_manned_json(obj_key.video_id, obj_key.obj_id, result->box, motorhs_algor.at(idx)); save_snapshot_process(obj_key, motorhs_algor.at(idx), src_img, roi_img, 0, json_str); } else { VPCUtil::vpc_img_release(src_img); VPCUtil::vpc_img_release(roi_img); } } } } if (task_param_ptr->nonmotor_vehicle_algors.find(algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE) != task_param_ptr->nonmotor_vehicle_algors.end()) { const auto &algor_other_params = task_other_params->find(algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE); const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; // auto result = motor_phoneprocess_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE}); auto result = motor_refit_phoneprocess_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE}); if (result.get()) { village_alarm = true; algorithm_types.push_back((int)algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE); video_folder = basic_param->video_folder; result_folder = basic_param->result_folder; result_folder_little = basic_param->result_folder_little; // 原图及抠图 vpc_img_info src_img; src_img.pic_desc = result->origin_img_desc; src_img.task_id = obj_key.video_id; vpc_img_info roi_img; roi_img.pic_desc = result->roi_img_desc; roi_img.task_id = obj_key.video_id; roi_img.object_id = obj_key.obj_id; if (save_single_algor_pic) { auto json_str = helpers::gen_json::gen_manned_json(obj_key.video_id, obj_key.obj_id, result->box, algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE); save_snapshot_process(obj_key, algorithm_type_t::NONMOTOR_VEHICLE_USEPHONE, src_img, roi_img, 0, json_str); } else { VPCUtil::vpc_img_release(src_img); VPCUtil::vpc_img_release(roi_img); } } } if (task_param_ptr->nonmotor_vehicle_algors.find(algorithm_type_t::NONMOTOR_VEHICLE_REFIT) != task_param_ptr->nonmotor_vehicle_algors.end()) { const auto &algor_other_params = task_other_params->find(algorithm_type_t::NONMOTOR_VEHICLE_REFIT); const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; auto result = motor_refitprocess_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algorithm_type_t::NONMOTOR_VEHICLE_REFIT}); // auto result = motor_refit_phoneprocess_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algorithm_type_t::NONMOTOR_VEHICLE_REFIT}); if (result.get()) { village_alarm = true; algorithm_types.push_back((int)algorithm_type_t::NONMOTOR_VEHICLE_REFIT); video_folder = basic_param->video_folder; result_folder = basic_param->result_folder; result_folder_little = basic_param->result_folder_little; // 原图及抠图 vpc_img_info src_img; src_img.pic_desc = result->origin_img_desc; src_img.task_id = obj_key.video_id; vpc_img_info roi_img; roi_img.pic_desc = result->roi_img_desc; roi_img.task_id = obj_key.video_id; roi_img.object_id = obj_key.obj_id; if (save_single_algor_pic) { auto json_str = helpers::gen_json::gen_manned_json(obj_key.video_id, obj_key.obj_id, result->box, algorithm_type_t::NONMOTOR_VEHICLE_REFIT); save_snapshot_process(obj_key, algorithm_type_t::NONMOTOR_VEHICLE_REFIT, src_img, roi_img, 0, json_str); } else { VPCUtil::vpc_img_release(src_img); VPCUtil::vpc_img_release(roi_img); } } } vector trafficlight_algor = {algorithm_type_t::PERSON_RUNNING_REDLIGHTS, algorithm_type_t::NONMOTOR_RUNNING_REDLIGHTS}; for (size_t idx = 0; idx < trafficlight_algor.size(); ++idx) { if (algor_param[task_id].count(trafficlight_algor.at(idx))) { const auto &algor_other_params = task_other_params->find(trafficlight_algor.at(idx)); const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; auto result = traffic_lightprocess_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, trafficlight_algor.at(idx)}); if (result.get()) { village_alarm = true; algorithm_types.push_back((int)trafficlight_algor.at(idx)); video_folder = basic_param->video_folder; result_folder = basic_param->result_folder; result_folder_little = basic_param->result_folder_little; // 原图及抠图 vpc_img_info src_img; src_img.pic_desc = result->origin_img_desc; src_img.task_id = obj_key.video_id; vpc_img_info roi_img; roi_img.pic_desc = result->roi_img_desc; roi_img.task_id = obj_key.video_id; roi_img.object_id = obj_key.obj_id; if (save_single_algor_pic) { auto json_str = helpers::gen_json::gen_manned_json(obj_key.video_id, obj_key.obj_id, result->box, trafficlight_algor.at(idx)); save_snapshot_process(obj_key, trafficlight_algor.at(idx), src_img, roi_img, 0, json_str); } else { VPCUtil::vpc_img_release(src_img); VPCUtil::vpc_img_release(roi_img); } } } } vector roadseg_algor = {algorithm_type_t::PERSON_IN_VEHICLELANE, algorithm_type_t::PERSON_CROSS, algorithm_type_t::NONMOTOR_IN_VEHICLELANE, algorithm_type_t::NONMOTOR_CEOSSPARKLINE, algorithm_type_t::NONMOTOR_WRONGDIRECTION, algorithm_type_t::VEHICLE_WRONGDIRECTION, algorithm_type_t::VEHICLE_NOTGIVEWAY, algorithm_type_t::VEHICLE_SOLIDLINETURNAROUND, algorithm_type_t::VEHICLE_NOTDECELERATION }; for (size_t idx = 0; idx < roadseg_algor.size(); ++idx) { if (algor_param[task_id].count(roadseg_algor.at(idx))) { const auto &algor_other_params = task_other_params->find(roadseg_algor.at(idx)); const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; auto result = roadseg_corrprocess_.get_result_by_objectid(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, roadseg_algor.at(idx)}); if (result.get()) { village_alarm = true; algorithm_types.push_back((int)roadseg_algor.at(idx)); video_folder = basic_param->video_folder; result_folder = basic_param->result_folder; result_folder_little = basic_param->result_folder_little; // 原图及抠图 vpc_img_info src_img; src_img.pic_desc = result->origin_img_desc; src_img.task_id = obj_key.video_id; vpc_img_info roi_img; roi_img.pic_desc = result->roi_img_desc; roi_img.task_id = obj_key.video_id; roi_img.object_id = obj_key.obj_id; if (save_single_algor_pic) { auto json_str = helpers::gen_json::gen_manned_json(obj_key.video_id, obj_key.obj_id, result->box, roadseg_algor.at(idx)); save_snapshot_process(obj_key, roadseg_algor.at(idx), src_img, roi_img, 0, json_str); } else { VPCUtil::vpc_img_release(src_img); VPCUtil::vpc_img_release(roi_img); } } } } m_TotalObjMtx.lock(); if (m_total_obj_info.find(obj_key) != m_total_obj_info.end()) { vector().swap(m_total_obj_info[obj_key].center_points); m_total_obj_info.erase(obj_key); } m_TotalObjMtx.unlock(); map _total_snapshot_info = m_snapshot_reprocessing->get_total_village_snapshot_info(); if(_total_snapshot_info.size() <= 0){ continue; } LOG_DEBUG("_total_snapshot_info size: {}", _total_snapshot_info.size()); auto it = _total_snapshot_info.find(obj_key); if (it == _total_snapshot_info.end()) { continue; } // 推送轨迹结束的目标用于数量统计 { const OBJ_VALUES obj_value = it->second; std::vector algo_results; std::vector algorithm_types; algorithm_types.push_back((int)algorithm_type_t::FLOW_STATISTICS); #ifdef POST_USE_RABBITMQ video_object_snapshot new_obj_ss_info; new_obj_ss_info.analysisRes = nullptr; new_obj_ss_info.object_id = obj_key.obj_id; new_obj_ss_info.obj_info.set_data(obj_value.snapShots[1].index.index, obj_value.snapShots[1].confidence, 0, 0, 0, 0); strcpy(new_obj_ss_info.task_id, obj_key.video_id.c_str()); strcpy(new_obj_ss_info.video_image_path, ""); strcpy(new_obj_ss_info.snapshot_image_path, ""); algo_results.push_back(new_obj_ss_info); auto json_str = helpers::gen_json::gen_village_json(task_id, obj_key.obj_id, algorithm_types, algo_results, ""); mq_manager_->publish(mq_type_t::ALARM_MQ, json_str.c_str(), true); #endif } if (village_alarm) { const OBJ_VALUES obj_value = it->second; std::string cur_timestamp_ms = std::to_string(helpers::timer::get_cur_time_ms()); // 视频 string video_file_name = video_folder + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + cur_timestamp_ms + ".mp4"; string json_str = ""; std::vector algo_results; for (int sp_idx = 0; sp_idx < 3; sp_idx ++) { int num = sp_idx + 1; // 原图 // LOG_DEBUG("原图"); std::string fpath_origin = result_folder + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + cur_timestamp_ms + "_picnum_" + std::to_string(num) + ".jpg"; ImgSaveInfo origin_save_info; origin_save_info.file_path = fpath_origin; origin_save_info.img_info = obj_value.snapShots[sp_idx].snapShot; origin_save_info.obj_rect = obj_value.snapShots[sp_idx].obj_pos; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(origin_save_info); // LOG_DEBUG("抠图"); // 抠图 string object_file_name = result_folder_little + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + cur_timestamp_ms + "_picnum_" + std::to_string(num) + ".jpg"; ImgSaveInfo obj_save_info; obj_save_info.file_path = object_file_name; obj_save_info.img_info = obj_value.snapShots[sp_idx].snapShotLittle; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(obj_save_info); #ifdef POST_USE_RABBITMQ video_object_snapshot new_obj_ss_info; new_obj_ss_info.analysisRes = nullptr; new_obj_ss_info.object_id = obj_key.obj_id; new_obj_ss_info.obj_info.set_data(obj_value.snapShots[sp_idx].index.index, obj_value.snapShots[sp_idx].confidence, obj_value.snapShots[sp_idx].obj_pos.left_, obj_value.snapShots[sp_idx].obj_pos.top_, obj_value.snapShots[sp_idx].obj_pos.left_ + obj_value.snapShots[sp_idx].obj_pos.width_, obj_value.snapShots[sp_idx].obj_pos.top_ + obj_value.snapShots[sp_idx].obj_pos.height_); strcpy(new_obj_ss_info.task_id, obj_key.video_id.c_str()); strcpy(new_obj_ss_info.video_image_path, fpath_origin.c_str()); strcpy(new_obj_ss_info.snapshot_image_path, object_file_name.c_str()); algo_results.push_back(new_obj_ss_info); #endif } #ifdef POST_USE_RABBITMQ json_str = helpers::gen_json::gen_village_json(task_id, obj_key.obj_id, algorithm_types, algo_results, video_file_name); #endif DeviceMemory* mem = vec_vptMem[i]; // string task_id = mem->getId(); RecoderInfo recoderInfo; recoderInfo.task_id = task_id; recoderInfo.object_id = std::to_string(j); recoderInfo.recoderPath = video_file_name; recoderInfo.frame_nb = mem->getFrameNb(); recoderInfo.mq_info = json_str; m_recoderinfo_queue_mtx.lock(); m_recoderinfo_queue.push_back(recoderInfo); m_recoderinfo_queue_mtx.unlock(); LOG_DEBUG("仅删除记录,显存会在保存线程中删除"); // 仅删除记录,显存会在保存线程中删除 m_snapshot_reprocessing->release_village_finished_locus_snapshot(obj_key.video_id, obj_key.obj_id, false); } else { // 删除记录,同时释放显存 m_snapshot_reprocessing->release_village_finished_locus_snapshot(obj_key.video_id, obj_key.obj_id, true); } } } } // for snapshot algorithm. 轨迹结束目标 做最后的结果返回(当前返回算法结果+快照保存路径) void CMultiSourceProcess::retrograde_snapshot(vector& vpt_interest_task_id, vector> deleteObjectID) { auto task_iter = vpt_interest_task_id.begin(); for (int i = 0; i < deleteObjectID.size(); i++, ++task_iter) // loop taskId. { for (int &j : deleteObjectID[i]) // loop algor type. { OBJ_KEY obj_key = {*task_iter, j}; auto task_param_ptr = m_task_param_manager->get_task_algor_param(obj_key.video_id); /* 开启行人&机动车逆行算法模块,获取该目标的算法分析结果 返回结果+快照 最后释放资源 */ if (task_param_ptr->human_algors.find(algorithm_type_t::PEDESTRIAN_RETROGRADE) != task_param_ptr->human_algors.end()){ retrograde_trespass_alarm(obj_key, algorithm_type_t::PEDESTRIAN_RETROGRADE); } if (task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_RETROGRADE) != task_param_ptr->vehicle_algors.end()){ retrograde_trespass_alarm(obj_key, algorithm_type_t::VEHICLE_RETROGRADE); } } } } // for snapshot algorithm. 轨迹结束目标 做最后的结果返回(当前返回算法结果+快照保存路径) void CMultiSourceProcess::trespass_snapshot(vector& vpt_interest_task_id, vector> deleteObjectID) { auto task_iter = vpt_interest_task_id.begin(); for (int i = 0; i < deleteObjectID.size(); i++, ++task_iter) // loop taskId. { for (int &j : deleteObjectID[i]) // loop algor type. { OBJ_KEY obj_key = {*task_iter, j}; auto task_param_ptr = m_task_param_manager->get_task_algor_param(obj_key.video_id); if (task_param_ptr->human_algors.find(algorithm_type_t::PEDESTRIAN_TRESPASS) != task_param_ptr->human_algors.end()){ retrograde_trespass_alarm(obj_key, algorithm_type_t::PEDESTRIAN_TRESPASS); } if (task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_TRESPASS) != task_param_ptr->vehicle_algors.end()){ retrograde_trespass_alarm(obj_key, algorithm_type_t::VEHICLE_TRESPASS); } } } } void CMultiSourceProcess::retrograde_trespass_alarm(const OBJ_KEY &obj_key, const algorithm_type_t &algor_type) { vector results ; if(algor_type == algorithm_type_t::PEDESTRIAN_RETROGRADE || algor_type == algorithm_type_t::VEHICLE_RETROGRADE){ results = pedestrian_vehicle_retrograde_.get_results_by_id(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algor_type}); } else if(algor_type == algorithm_type_t::PEDESTRIAN_TRESPASS || algor_type == algorithm_type_t::VEHICLE_TRESPASS){ results = pedestrian_vehicle_trespass_.get_results_by_id(ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algor_type}); } else { return ; } bool bRetroGrade = true ; for (unsigned idx = 0; idx < results.size(); ++idx) { auto &result = results[idx]; vpc_img_info src_img; src_img.pic_desc = result.origin_img_desc; src_img.task_id = obj_key.video_id; vpc_img_info roi_img; roi_img.pic_desc = result.roi_img_desc; roi_img.task_id = obj_key.video_id; roi_img.object_id = obj_key.obj_id; if(bRetroGrade){ auto &&json_str = helpers::gen_json::gen_retrograde_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); save_snapshot_process(obj_key, algor_type, src_img, roi_img, idx, json_str); } else { VPCUtil::vpc_img_release(src_img); VPCUtil::vpc_img_release(roi_img); } if (bRetroGrade == true) { bRetroGrade = false; } } } bool CMultiSourceProcess::save_snapshot_process(const OBJ_KEY &obj_key, const algorithm_type_t &algorithm_type, vpc_img_info src_img, vpc_img_info roi_img, const long long id,const std::string &json_str) { auto task_other_params = m_task_param_manager->get_task_other_param(obj_key.video_id); const auto &algor_other_params = task_other_params->find(algorithm_type); if (algor_other_params == task_other_params->end()) { LOG_ERROR("task_id {} not found {} error", obj_key.video_id, int(algorithm_type)); return false; } const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; std::string cur_time1 = std::to_string(helpers::timer::get_timestamp()); const std::string fpath_origin = basic_param->result_folder + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + std::to_string(id) + "_" + cur_time1 + ".jpg"; ImgSaveInfo obj_save_info; obj_save_info.file_path = fpath_origin; obj_save_info.img_info = src_img; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(obj_save_info); { // 抠图保存 std::string cur_time2 = std::to_string(helpers::timer::get_timestamp()); const std::string fpath_roi = basic_param->result_folder_little + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + std::to_string(id) + "_" + cur_time2 + ".jpg"; // 调用快照保存后处理模块 将快照保存 ImgSaveInfo obj_save_info; obj_save_info.file_path = fpath_roi; obj_save_info.img_info = roi_img; obj_save_info.json_str = json_str; #ifdef POST_USE_RABBITMQ if(json_str.length() > 0) { auto json_str_tmp = helpers::string::replace_all(json_str, helpers::gen_json::ORI_IMAGE_PATH_PLACEHOLDER, fpath_origin); obj_save_info.json_str = helpers::string::replace_all(json_str_tmp, helpers::gen_json::ROI_IMAGE_PATH_PLACEHOLDER, fpath_roi); } #endif m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(obj_save_info); } return true; } // 人脸检测抓拍算法模块 void CMultiSourceProcess::algorthim_face_detect(vector vec_gpuMem) { vector interest_task_list; vector interest_imgs; vector vec_vptMem; for (int i = 0; i < vec_gpuMem.size(); i++) { DeviceMemory* mem = vec_gpuMem[i]; if (!m_task_param_manager->task_has_face_algor(mem->getId())){ continue; } sy_img img; img.w_ = mem->getWidth(); img.h_ = mem->getHeight(); img.c_ = mem->getChannel(); img.data_ = mem->getMem(); interest_imgs.push_back(img); interest_task_list.push_back(mem->getId()); vec_vptMem.push_back(mem); } if (!interest_imgs.empty()) { unsigned image_size = interest_imgs.size(); // 人脸检测、跟踪 std::vector facedet_result(image_size); std::vector> face_deleteObjectID(image_size); int ret = m_face_det_ai_engine.ai_engine_process_batch(interest_task_list, interest_imgs.data(), facedet_result, face_deleteObjectID); if(ret <= 0){ LOG_ERROR("face detect error!!!"); return; } // 跟踪结果送入快照更新 m_snapshot_reprocessing->update_face_bestsnapshot(vec_vptMem, facedet_result, face_deleteObjectID); // 保存已结束轨迹的目标 auto task_iter_face = interest_task_list.begin(); //debug by zsh for (int i = 0; i < face_deleteObjectID.size(); i++) { for (int j = 0; j < face_deleteObjectID[i].size(); ++j) { OBJ_KEY deleteObj = {*task_iter_face, face_deleteObjectID[i][j]}; LOG_TRACE("{}: {}",*task_iter_face,face_deleteObjectID[i][j]); face_locus_finished(deleteObj); } ++task_iter_face; } for (int i = 0; i < face_deleteObjectID.size(); ++i){ std::vector().swap(face_deleteObjectID[i]); } std::vector>().swap(face_deleteObjectID); std::vector().swap(facedet_result); } } void CMultiSourceProcess::face_locus_finished(const OBJ_KEY obj_key) { map _total_face_snapshot_info = m_snapshot_reprocessing->get_total_face_snapshot_info(); auto it = _total_face_snapshot_info.find(obj_key); if ( it == _total_face_snapshot_info.end()) { return; } LOG_DEBUG("face reprocessing {}:{}.", obj_key.video_id, obj_key.obj_id); auto task_other_params = m_task_param_manager->get_task_other_params(); const algor_basic_config_param_t *cur_param = ((algor_init_config_param_t *)(task_other_params[obj_key.video_id][algorithm_type_t::FACE_SNAPSHOT]))->basic_param; OBJ_VALUE obj_value = it->second; std::string cur_time1 = std::to_string(helpers::timer::get_timestamp()); // 原图 string fpath_src = std::string(cur_param->result_folder) + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + cur_time1 + ".jpg"; ImgSaveInfo origin_save_info; origin_save_info.file_path = fpath_src; origin_save_info.img_info = obj_value.snapShot; origin_save_info.obj_rect = obj_value.obj_pos; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(origin_save_info); // 抠图 string json_str = ""; std::string cur_time2 = std::to_string(helpers::timer::get_timestamp()); string fpath_snapShotLittle = std::string(cur_param->result_folder_little) + helpers::os::sep + obj_key.video_id + "_" + std::to_string(obj_key.obj_id) + "_" + cur_time2 + ".jpg"; #ifdef POST_USE_RABBITMQ json_str = helpers::gen_json::gen_face_detection_json( obj_key.video_id, obj_key.obj_id, fpath_snapShotLittle, fpath_src, obj_value.position, obj_value.confidence, obj_value.landmark_point, 25); #endif ImgSaveInfo roi_save_info; roi_save_info.file_path = fpath_snapShotLittle; roi_save_info.img_info = obj_value.snapShotLittle; roi_save_info.json_str = json_str; m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(roi_save_info); // 删除结束轨迹的数据 m_snapshot_reprocessing->release_finished_face_locus_snapshot(obj_key.video_id, obj_key.obj_id, false); } int CMultiSourceProcess::recode_thread() { LOG_INFO("recode_thread start..."); while(true) { if(m_bfinish){ break; } m_recoderinfo_queue_mtx.lock(); if(m_recoderinfo_queue.size() <= 0) { m_recoderinfo_queue_mtx.unlock(); std::this_thread::sleep_for(std::chrono::milliseconds(5)); continue; } RecoderInfo info = m_recoderinfo_queue.front(); m_recoderinfo_queue.pop_front(); m_recoderinfo_queue_mtx.unlock(); DecoderManager* pDecManager = DecoderManager::getInstance(); pDecManager->doRecode(info); } LOG_INFO("recode_thread end."); } bool CMultiSourceProcess::CheckTime() { struct tm* info; int nYear, nMonth, nDay; time_t raw; time(&raw); info = localtime(&raw); nYear = info->tm_year + 1900; nMonth = info->tm_mon + 1; nDay = info->tm_mday; if (nYear == 2024 && nMonth <= 12) { return true; } else { return false; } } bool CMultiSourceProcess::CheckLabel(int devId) { const char *token_password = "Capture_atlas_arm_2023"; std::string guid = std::string("Capture_atlas_arm")+std::to_string(devId); atlas_licence_param lic_param; memset(&lic_param, 0, sizeof(atlas_licence_param)); sprintf(lic_param.product_id, "%s", productSN); sprintf(lic_param.guid, "%s",guid.c_str()); sprintf(lic_param.token_pwd, "%s", token_password); lic_param.channel_num = 1; atlas_licence_token param_token; memset(¶m_token, 0, sizeof(atlas_licence_token)); int result = atlas_licence_connect(&(skt_handle), lic_param, ¶m_token); //printf("result:%d,code:%d,msg:%s \n", result, param_token.code, param_token.msg); //printf("token:%s\n", param_token.token); std::string recv_token = std::string(param_token.token); //atlas_licence_check_param check_param; memset(&(check_param), 0, sizeof(atlas_licence_check_param)); sprintf(check_param.token_pwd, "%s", token_password); sprintf(check_param.time, "%s", "2023-01-10 20:00:00"); sprintf(check_param.token, "%s", recv_token .c_str()); check_param.consume = 2; //授权check------------ check_label = -1;//初始值 std::cout << "sy_licence_check start." << std::endl; atlas_licence_check_result check_result; memset(&check_result, 0, sizeof(atlas_licence_check_result)); check_result.code = -1; int res = atlas_licence_check(skt_handle, check_param, &check_result); if(res!=0) { std::cout << "sy_licence_check failed." << std::endl; return false; } //std::cout << "code:" << check_result.code << ",msg:" << check_result.msg << std::endl; if(check_result.code!=0) { std::cout << "code:" << check_result.code << ",msg:" << check_result.msg << std::endl; return false; } std::cout << "sy_licence_check end." << std::endl; check_label = 0;//授权成功 return true; } bool CMultiSourceProcess::UpdateLabel() { //获取系统时间,每个月1号check一次授权 struct tm* info; int nYear, nMonth, nDay; time_t raw; time(&raw); info = localtime(&raw); nYear = info->tm_year + 1900; nMonth = info->tm_mon + 1; nDay = info->tm_mday; if(nDay==1) { if(check_label ==0) { LOG_INFO("atlas_licence_check start."); atlas_licence_check_result check_result; memset(&check_result, 0, sizeof(atlas_licence_check_result)); check_result.code = -1; int res = atlas_licence_check(skt_handle, check_param, &check_result); if(res!=0) { LOG_FATAL("sy_licence_check failed."); return false; } if(check_result.code!=0) { check_label = -1; LOG_FATAL("atlas_licence_check code:{}, msg:{}",check_result.code, check_result.msg); return false; } LOG_INFO("atlas_licence_check end."); check_label =1; } } else { check_label =0; } return true; }