From 28ace2795b473bad2dd11a22990590e141e7a3be Mon Sep 17 00:00:00 2001 From: 2657262686@qq.com <2657262686@qq.com> Date: Thu, 27 Oct 2022 17:56:13 +0800 Subject: [PATCH] 完善 WITH_SECOND_PROCESS --- tsl_aiplatform/ai_engine_module/fight_fall_cls.cpp | 252 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ tsl_aiplatform/ai_engine_module/fight_fall_cls.hpp | 3 +++ tsl_aiplatform/ai_platform/MultiSourceProcess.cpp | 59 +++++++++++++++++++++++++++++++++++++++-------------------- tsl_aiplatform/ai_platform/MultiSourceProcess.h | 8 ++++---- 4 files changed, 298 insertions(+), 24 deletions(-) diff --git a/tsl_aiplatform/ai_engine_module/fight_fall_cls.cpp b/tsl_aiplatform/ai_engine_module/fight_fall_cls.cpp index e055447..cedcf86 100644 --- a/tsl_aiplatform/ai_engine_module/fight_fall_cls.cpp +++ b/tsl_aiplatform/ai_engine_module/fight_fall_cls.cpp @@ -348,6 +348,258 @@ namespace ai_engine_module } return true; } // func end + + bool FightfallCls::process_mstreams2(const std::vector &taskIds, const sy_img *det_input_images, const std::vector &det_results, + std::vector &results) + { + if (!check_initied()) + return false; + + if (det_results.empty()) + { + LOG_ERROR("[FightfallCls] call init function please."); + LOG_ERROR("detection result is empty."); + return false; + } + + int n_images = det_results.size(); // or n_stream + + unsigned flattened_idx = 0; + std::map flattened_idx_to_batch_idx; + + + /* 1. Crop & keep some interest class. */ + /* 1. preprocess. */ + auto taskId_iter = taskIds.begin(); + std::vector flattened_imgs(0); + std::vector flattened_interest_data(0); // + for (int n = 0; n < n_images; ++n) + { + int n_interest_obj = 0; + const sy_img &src_img = det_input_images[n]; + auto &boxes_of_one_image = det_results[n].obj; + for (int i = 0; i < det_results[n].obj_count; ++i) + { + auto &box = boxes_of_one_image[i]; + if (static_cast(box.index) == det_class_label_t::HUMAN) + { + auto &taskId = *taskId_iter; + + input_data_wrap_t data; + int top = std::max(int(box.top - (IMAGE_CROP_EXPAND_RATIO * box.top)), 0); + int left = std::max(int(box.left - (IMAGE_CROP_EXPAND_RATIO * box.left)), 0); + int right = std::min(int(box.right + (IMAGE_CROP_EXPAND_RATIO * box.right)), src_img.w_); + int bottom = std::min(int(box.bottom + (IMAGE_CROP_EXPAND_RATIO * box.bottom)), src_img.h_); + + int width = right - left; + int height = bottom - top; + + auto fall_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fall_algor_type_); + auto fall_algor_param = (fall_algor_param_wrap != nullptr) ? ((fall_algor_param_type)fall_algor_param_wrap->algor_param) : nullptr; + auto fall_basic_param = (fall_algor_param_wrap != nullptr) ? (fall_algor_param_wrap->basic_param) : nullptr; + + auto fight_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fight_algor_type_); + auto fight_algor_param = (fight_algor_param_wrap != nullptr) ? ((fight_algor_param_type)fight_algor_param_wrap->algor_param) : nullptr; + auto fight_basic_param = (fight_algor_param_wrap != nullptr) ? (fight_algor_param_wrap->basic_param) : nullptr; + + + auto minimum_width = std::min((fall_algor_param == nullptr ? DEFAULT_MIN_WIDTH : fall_algor_param->pedestrian_min_width), + (fight_algor_param == nullptr ? DEFAULT_MIN_WIDTH : fight_algor_param->pedestrian_min_width)); + auto minimum_height = std::min((fall_algor_param == nullptr ? DEFAULT_MIN_HEIGHT : fall_algor_param->pedestrian_min_height), + (fight_algor_param == nullptr ? DEFAULT_MIN_HEIGHT : fight_algor_param->pedestrian_min_height)); + auto minimum_threshold = std::min((fall_algor_param == nullptr ? DEFAULT_PTHRESHOLD : fall_algor_param->threshold), + (fight_algor_param == nullptr ? DEFAULT_PTHRESHOLD : fight_algor_param->threshold)); + + sy_rect intel_rect; + if (fight_basic_param == nullptr) + { + if (fall_basic_param != nullptr) + intel_rect = fall_basic_param->algor_valid_rect; + } + else + { + intel_rect = fight_basic_param->algor_valid_rect; + } + + + if ((width < minimum_width || height < minimum_height || box.confidence < minimum_threshold) || + !snapshot_legal_inarea(intel_rect, left, top, right, bottom)) + continue; + + data.box.top = top; + data.box.left = left; + data.box.right = right; + data.box.bottom = bottom; + data.box.score = box.confidence; + data.taskId = taskId; + data.objId = box.id; + + + sy_img img; + img.w_ = width; + img.h_ = height; + img.c_ = src_img.c_; + + cudaError_t cuda_status; + const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char); + if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void**)&img.data_, nbytes))) + { + LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ", cudaGetErrorString(cuda_status), nbytes, nbytes / (1024 * 1024)); + continue; + } + + if (CUDA_SUCCESS != ( cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, left, top, width, height))) + { + LOG_ERROR("Crop image GPU failed error is %s wh is [{}, {}] ltrb is [{} {} {} {}]", + cudaGetErrorString(cuda_status), src_img.w_, src_img.h_, data.box.left, data.box.top, data.box.right, data.box.bottom); + CHECK(cudaFree(img.data_)); + continue; + } + flattened_imgs.emplace_back(std::move(img)); + flattened_interest_data.emplace_back(std::move(data)); + flattened_idx_to_batch_idx[flattened_idx++] = n; + } + } + ++taskId_iter; + } + + + /* 2. inference. */ + int n_input_image = flattened_imgs.size(); + fight_det_result model_results[n_input_image]; + { + int steps = (n_input_image + MAX_BATCH - 1) / MAX_BATCH; + + for (int step = 0; step < steps; ++step) + { + int offset = step * MAX_BATCH; + int batch_size = (step == steps - 1) ? n_input_image - offset : MAX_BATCH; + fight_det_process_batch(tools_, flattened_imgs.data() + offset, batch_size, model_results + offset); + } + } + + + /* 3. postprocess. */ + { + /* a. review to 2d format. */ + std::unordered_map> taskid_to_sdk_result; + { + for (int n = 0; n < n_input_image; ++n) + taskid_to_sdk_result[flattened_interest_data[n].taskId].emplace_back(std::move(model_results[n])); + } + + /* b. post process */ + int n = 0; + for (auto iter = taskIds.begin(); iter != taskIds.end(); ++iter) + { + result_data_t result_data; + auto &taskId = *iter; + + auto fall_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fall_algor_type_); + auto fall_algor_param = (fall_algor_param_wrap != nullptr) ? ((fall_algor_param_type)fall_algor_param_wrap->algor_param) : nullptr; + + auto fight_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fight_algor_type_); + auto fight_algor_param = (fight_algor_param_wrap != nullptr) ? ((fight_algor_param_type)fight_algor_param_wrap->algor_param) : nullptr; + + auto &model_result = taskid_to_sdk_result[taskId]; + const unsigned model_result_size = model_result.size(); + + bool has_fight_pair[model_result_size]; + memset(has_fight_pair, false, model_result_size * sizeof(bool)); + + for (int i = 0; i < model_result_size; ++i, ++n) + { + const sy_img& src_img = det_input_images[flattened_idx_to_batch_idx[n]]; + auto &cropped_img = flattened_imgs[n]; // croped image. + auto &preprocessed_data = flattened_interest_data[n]; + auto &fight_result_iter = model_result[i]; + for (int j = 0; j < FIGHT_MODEL_NUM; ++j) + { + auto &res = fight_result_iter.fight_infos[j]; + + if (fight_algor_param != nullptr) + if (!has_fight_pair[i]) + for (int z = i + 1; z < model_result_size; ++z) + if (!has_fight_pair[z] && (res.fight_score > fight_algor_param->threshold || + model_result[z].fight_infos[j].fight_score > fight_algor_param->threshold)) + if (iou(preprocessed_data.box, flattened_interest_data[n + (z - i)].box) > fight_algor_param->iou_threshold) + { + // std::printf("[Debug] fight Pair "); + has_fight_pair[i] = has_fight_pair[z] = true; + + auto &preprocessed_data2 = flattened_interest_data[n + (z - i)]; + result_fight_data_t data; + { + data.taskid = taskId; + data.objectids.insert(preprocessed_data.objId); + data.objectids.insert(preprocessed_data2.objId); + data.box = union_box(preprocessed_data.box, preprocessed_data2.box); + // data.box.score = (res.fight_score + model_result[z].fight_infos[j].fight_score) * 0.5; + data.box.score = std::max(res.fight_score, model_result[z].fight_infos[j].fight_score); + + // std::printf("\tltrb is [%d %d %d %d] wh of src img is [%d %d]", data.box.left, data.box.top, data.box.right, data.box.bottom, src_img.w_, src_img.h_); + + sy_img img; + { + img.c_ = src_img.c_; + img.w_ = data.box.width(); + img.h_ = data.box.height(); + } + cudaError_t cuda_status; + const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char); + if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void**)&img.data_, nbytes))) + { + LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ", cudaGetErrorString(cuda_status), nbytes, nbytes / (1024 * 1024)); + continue; + } + + if (CUDA_SUCCESS != ( cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, data.box.left, data.box.top, img.w_, img.h_))) + { + LOG_ERROR("Crop image GPU failed error is {} wh is [{} {}] ltrb is [{} {} {} {}]", + cudaGetErrorString(cuda_status), src_img.w_, src_img.h_, data.box.left, data.box.top, data.box.right, data.box.bottom); + CHECK(cudaFree(img.data_)); + continue; + } + data.roi_img = img; + data.ori_img = src_img; + data.id = gid_++; + } + result_data.fight_data.emplace_back(std::move(data)); + } + + if (fall_algor_param != nullptr) + if (res.fall_score > fall_algor_param->threshold) + { + if ((float)(src_img.h_ - preprocessed_data.box.bottom) > (float)(src_img.h_ * 0.05)) + { + result_fall_data_t data; + { + data.box = preprocessed_data.box; + data.objectid = preprocessed_data.objId; + data.taskid = taskId; + data.roi_img = cropped_img; + data.ori_img = src_img; + data.id = gid_++; + } + result_data.fall_data.emplace_back(std::move(data)); + goto _continue; + } + } + + + } + cudaFree(cropped_img.data_); + _continue: + { + + } + } + results.emplace_back(std::move(result_data)); + } + } + return true; + } // func end + } // namespace fight_fall_cls } // namespace ai_engine_module diff --git a/tsl_aiplatform/ai_engine_module/fight_fall_cls.hpp b/tsl_aiplatform/ai_engine_module/fight_fall_cls.hpp index 7d11310..077c9fa 100644 --- a/tsl_aiplatform/ai_engine_module/fight_fall_cls.hpp +++ b/tsl_aiplatform/ai_engine_module/fight_fall_cls.hpp @@ -107,6 +107,9 @@ namespace ai_engine_module bool process_mstreams(const std::set &taskIds, const sy_img *det_input_images, const std::vector &det_result, std::vector &results); + bool process_mstreams2(const std::vector &taskIds, const sy_img *det_input_images, const std::vector &det_result, + std::vector &results); + FightfallCls(const FightfallCls&) = delete; FightfallCls& operator=(const FightfallCls&) = delete; diff --git a/tsl_aiplatform/ai_platform/MultiSourceProcess.cpp b/tsl_aiplatform/ai_platform/MultiSourceProcess.cpp index 3b5b4ba..6fc232f 100644 --- a/tsl_aiplatform/ai_platform/MultiSourceProcess.cpp +++ b/tsl_aiplatform/ai_platform/MultiSourceProcess.cpp @@ -249,27 +249,46 @@ bool CMultiSourceProcess::add_task_operation(task_param _cur_task_param){ // pDecManager->setDecKeyframe(config.name, true); // 只对关键帧解码 pDecManager->startDecodeByName(config.name); + const char* task_id = _cur_task_param.task_id; + + // 保存新添加任务的配置参数 + m_task_param_manager->add_task_param(task_id, _cur_task_param); + int input_image_width = 0, input_image_height = 0; pDecManager->getResolution(config.name, input_image_width, input_image_height); +#ifdef WITH_SECOND_PROCESS + /* 如果开启了行人 机动车非法闯入功能 生成闯入区域mask */ + auto new_task_algor_param = m_task_param_manager->get_task_other_param(task_id); - const char *task_id = _cur_task_param.task_id; + if (new_task_algor_param->find(algorithm_type_t::PEDESTRIAN_TRESPASS) != new_task_algor_param->end()) { + pedestrian_vehicle_trespass_.pedestrianvehicletrespass_init_region( + task_id, algorithm_type_t::PEDESTRIAN_TRESPASS, input_image_width, input_image_height); + } - m_task_param_manager->add_task_param(task_id, _cur_task_param); + if (new_task_algor_param->find(algorithm_type_t::VEHICLE_TRESPASS) != new_task_algor_param->end()) { + pedestrian_vehicle_trespass_.pedestrianvehicletrespass_init_region( + task_id, algorithm_type_t::VEHICLE_TRESPASS, input_image_width, input_image_height); + } +#endif + + ((save_snapshot_reprocessing *)m_save_snapshot_reprocessing)->add_newtask(task_id); -// // 人车物跟踪 -// if (task_has_vpt_algor(task_id)) -// AddTaskTracker(VPT_Handle_, cur_task.first); + // 人车物跟踪 + if (task_has_vpt_algor(task_id)) + AddTaskTracker(VPT_Handle_, task_id); -// // 人脸跟踪 -// #ifdef WITH_FACE_DET_SS -// if (task_has_face_algor(task_id)) -// m_face_det_ai_engine.operator_tracker(task_id, ADDTASK, SKIP_FRAME); // 跳帧数暂时写死 -// #endif + // 人脸跟踪 +#ifdef WITH_FACE_DET_SS + if (task_has_face_algor(task_id)) + m_face_det_ai_engine.operator_tracker(task_id, ADDTASK, SKIP_FRAME); // 跳帧数暂时写死 +#endif + // 启动算法处理线程 startProcessByGpuid(_cur_task_param.gpu_id_); } +// 启动算法处理线程 void CMultiSourceProcess::startProcessByGpuid(const string gpuid){ struct ThreadArg{ @@ -298,7 +317,7 @@ void CMultiSourceProcess::startProcessByGpuid(const string gpuid){ } void CMultiSourceProcess::post_decode_thread(task_param _cur_task_param, AVFrame * gpuFrame){ - if (gpuFrame->format == AV_PIX_FMT_CUDA){ + if (gpuFrame->format == AV_PIX_FMT_CUDA){ GpuRgbMemory* gpuMem = new GpuRgbMemory(3, gpuFrame->width, gpuFrame->height, _cur_task_param.task_id, _cur_task_param.gpu_id_ , true); cudaSetDevice(atoi(_cur_task_param.gpu_id_)); @@ -530,16 +549,16 @@ void CMultiSourceProcess::algorthim_pedestrian_safety(vector& vpt_intere } // 逆行&非法闯入算法模块 -void CMultiSourceProcess::algorthim_retrograde_trespass(vector& vpt_interest_task_id, vector& vpt_interest_imgs, vector& vptResult - ,vector>& deleteObjectID){ +void CMultiSourceProcess::algorthim_retrograde_trespass(vector& vpt_interest_task_id, vector& vpt_interest_imgs + , vector& vptResult ,vector>& deleteObjectID){ #ifdef WITH_SECOND_PROCESS vector interest_task_id; - decltype(vptResult) interest_vpt_result; + vector interest_vpt_result; vector interest_imgs(0); vector trespass_interest_task_id; - decltype(vptResult) trespass_interest_vpt_result; - decltype(deleteObjectID) trespass_interest_deleteobjs; + vector trespass_interest_vpt_result; + vector> trespass_interest_deleteobjs; vector trespass_interest_imgs(0); int _idx = 0; @@ -847,7 +866,7 @@ void CMultiSourceProcess::algorithm_fight_fall(vector& vpt_interest_task /* b. process result. */ if (!interest_imgs.empty()) { std::vector results; - fight_fall_cls_.process_mstreams(interest_task_id, interest_imgs.data(), interest_vpt_result, results); + fight_fall_cls_.process_mstreams2(interest_task_id, interest_imgs.data(), interest_vpt_result, results); for (auto &result : results) { for (auto &res : result.fight_data) { auto &taskId = res.taskid; @@ -878,7 +897,7 @@ void CMultiSourceProcess::algorithm_takeaway_member_cls(vector& vpt_inte #ifdef WITH_SECOND_PROCESS /* find takeaway member classification taskId. */ vector interest_task_id; - decltype(vptResult) interest_vpt_result; + vector interest_vpt_result; vector interest_imgs(0); int _idx = 0; @@ -887,7 +906,7 @@ void CMultiSourceProcess::algorithm_takeaway_member_cls(vector& vpt_inte { const auto &task_id = *_task_id_iter; - auto algor_map = pThreadParam->m_task_param_manager->get_task_other_param(task_id); + auto algor_map = m_task_param_manager->get_task_other_param(task_id); if (algor_map->find(algorithm_type_t::TAKEAWAY_MEMBER_CLASSIFICATION) == algor_map->end()) continue; @@ -907,7 +926,7 @@ int CMultiSourceProcess::everyframe_process(vector &task_in_play_id, sy_ vector &ol_det_result) { #ifdef WITH_SECOND_PROCESS /* 人数聚集算法功能 每帧都会获取算法结果 并返回 */ - auto results = m_human_gather_statistics::human_gather_statistics_process2(task_in_play_id, images, ol_det_result); + auto results = m_human_gather_statistics.human_gather_statistics_process2(task_in_play_id, images, ol_det_result); for (auto &result : results) { #ifdef POST_USE_RABBITMQ diff --git a/tsl_aiplatform/ai_platform/MultiSourceProcess.h b/tsl_aiplatform/ai_platform/MultiSourceProcess.h index 76f5b04..45d8d23 100644 --- a/tsl_aiplatform/ai_platform/MultiSourceProcess.h +++ b/tsl_aiplatform/ai_platform/MultiSourceProcess.h @@ -210,10 +210,10 @@ private: int endframe_obj_process(const OBJ_KEY &obj_key, algorithm_type_t algor_type); /* 实现快照保存功能（还未真正保存 将显存图片cp到内存 * 直接保存本地或者存入缓存队列异步保存，保存方式看需求，报警类需要同步保存报警，分析类可异步保存后返回）*/ - bool save_snapshot_process(const OBJ_KEY &obj_key, const algorithm_type_t &algorithm_type, - const sy_img &ori_img, const sy_img &roi_img, const long long id, - const std::string &json_str, bool enable_async, - const bool ori_img_is_in_gpu, const bool roi_img_is_in_gpu); + bool save_snapshot_process(const OBJ_KEY &obj_key, const algorithm_type_t &algorithm_type, const sy_img &ori_img, + const sy_img &roi_img, const long long id, const std::string &json_str, + bool enable_async = true, const bool ori_img_is_in_gpu = true, + const bool roi_img_is_in_gpu = true); private: map gpuProcessthreadMap; -- libgit2 0.21.4