Hu Chunming / src

Browse Code »

Commit 28ace2795b473bad2dd11a22990590e141e7a3be

Authored by Hu Chunming 2022-10-27 17:56:13 +0800

1 parent 6e2b079e

完善 WITH_SECOND_PROCESS

Inline Side-by-side

Showing 4 changed files with 298 additions and 24 deletions

tsl_aiplatform/ai_engine_module/fight_fall_cls.cpp

View file @28ace27

...	...	@@ -348,6 +348,258 @@ namespace ai_engine_module
348	348	}
349	349	return true;
350	350	} // func end
	351	+
	352	+ bool FightfallCls::process_mstreams2(const std::vector<taskid_t> &taskIds, const sy_img *det_input_images, const std::vector<onelevel_det_result> &det_results,
	353	+ std::vector<result_data_t> &results)
	354	+ {
	355	+ if (!check_initied())
	356	+ return false;
	357	+
	358	+ if (det_results.empty())
	359	+ {
	360	+ LOG_ERROR("[FightfallCls] call init function please.");
	361	+ LOG_ERROR("detection result is empty.");
	362	+ return false;
	363	+ }
	364	+
	365	+ int n_images = det_results.size(); // or n_stream
	366	+
	367	+ unsigned flattened_idx = 0;
	368	+ std::map<int, int> flattened_idx_to_batch_idx;
	369	+
	370	+
	371	+ /* 1. Crop & keep some interest class. */
	372	+ /* 1. preprocess. */
	373	+ auto taskId_iter = taskIds.begin();
	374	+ std::vector<sy_img> flattened_imgs(0);
	375	+ std::vector<input_data_wrap_t> flattened_interest_data(0); //
	376	+ for (int n = 0; n < n_images; ++n)
	377	+ {
	378	+ int n_interest_obj = 0;
	379	+ const sy_img &src_img = det_input_images[n];
	380	+ auto &boxes_of_one_image = det_results[n].obj;
	381	+ for (int i = 0; i < det_results[n].obj_count; ++i)
	382	+ {
	383	+ auto &box = boxes_of_one_image[i];
	384	+ if (static_cast<det_class_label_t>(box.index) == det_class_label_t::HUMAN)
	385	+ {
	386	+ auto &taskId = *taskId_iter;
	387	+
	388	+ input_data_wrap_t data;
	389	+ int top = std::max(int(box.top - (IMAGE_CROP_EXPAND_RATIO * box.top)), 0);
	390	+ int left = std::max(int(box.left - (IMAGE_CROP_EXPAND_RATIO * box.left)), 0);
	391	+ int right = std::min(int(box.right + (IMAGE_CROP_EXPAND_RATIO * box.right)), src_img.w_);
	392	+ int bottom = std::min(int(box.bottom + (IMAGE_CROP_EXPAND_RATIO * box.bottom)), src_img.h_);
	393	+
	394	+ int width = right - left;
	395	+ int height = bottom - top;
	396	+
	397	+ auto fall_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fall_algor_type_);
	398	+ auto fall_algor_param = (fall_algor_param_wrap != nullptr) ? ((fall_algor_param_type)fall_algor_param_wrap->algor_param) : nullptr;
	399	+ auto fall_basic_param = (fall_algor_param_wrap != nullptr) ? (fall_algor_param_wrap->basic_param) : nullptr;
	400	+
	401	+ auto fight_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fight_algor_type_);
	402	+ auto fight_algor_param = (fight_algor_param_wrap != nullptr) ? ((fight_algor_param_type)fight_algor_param_wrap->algor_param) : nullptr;
	403	+ auto fight_basic_param = (fight_algor_param_wrap != nullptr) ? (fight_algor_param_wrap->basic_param) : nullptr;
	404	+
	405	+
	406	+ auto minimum_width = std::min((fall_algor_param == nullptr ? DEFAULT_MIN_WIDTH : fall_algor_param->pedestrian_min_width),
	407	+ (fight_algor_param == nullptr ? DEFAULT_MIN_WIDTH : fight_algor_param->pedestrian_min_width));
	408	+ auto minimum_height = std::min((fall_algor_param == nullptr ? DEFAULT_MIN_HEIGHT : fall_algor_param->pedestrian_min_height),
	409	+ (fight_algor_param == nullptr ? DEFAULT_MIN_HEIGHT : fight_algor_param->pedestrian_min_height));
	410	+ auto minimum_threshold = std::min((fall_algor_param == nullptr ? DEFAULT_PTHRESHOLD : fall_algor_param->threshold),
	411	+ (fight_algor_param == nullptr ? DEFAULT_PTHRESHOLD : fight_algor_param->threshold));
	412	+
	413	+ sy_rect intel_rect;
	414	+ if (fight_basic_param == nullptr)
	415	+ {
	416	+ if (fall_basic_param != nullptr)
	417	+ intel_rect = fall_basic_param->algor_valid_rect;
	418	+ }
	419	+ else
	420	+ {
	421	+ intel_rect = fight_basic_param->algor_valid_rect;
	422	+ }
	423	+
	424	+
	425	+ if ((width < minimum_width \|\| height < minimum_height \|\| box.confidence < minimum_threshold) \|\|
	426	+ !snapshot_legal_inarea(intel_rect, left, top, right, bottom))
	427	+ continue;
	428	+
	429	+ data.box.top = top;
	430	+ data.box.left = left;
	431	+ data.box.right = right;
	432	+ data.box.bottom = bottom;
	433	+ data.box.score = box.confidence;
	434	+ data.taskId = taskId;
	435	+ data.objId = box.id;
	436	+
	437	+
	438	+ sy_img img;
	439	+ img.w_ = width;
	440	+ img.h_ = height;
	441	+ img.c_ = src_img.c_;
	442	+
	443	+ cudaError_t cuda_status;
	444	+ const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char);
	445	+ if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void**)&img.data_, nbytes)))
	446	+ {
	447	+ LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ", cudaGetErrorString(cuda_status), nbytes, nbytes / (1024 * 1024));
	448	+ continue;
	449	+ }
	450	+
	451	+ if (CUDA_SUCCESS != ( cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, left, top, width, height)))
	452	+ {
	453	+ LOG_ERROR("Crop image GPU failed error is %s wh is [{}, {}] ltrb is [{} {} {} {}]",
	454	+ cudaGetErrorString(cuda_status), src_img.w_, src_img.h_, data.box.left, data.box.top, data.box.right, data.box.bottom);
	455	+ CHECK(cudaFree(img.data_));
	456	+ continue;
	457	+ }
	458	+ flattened_imgs.emplace_back(std::move(img));
	459	+ flattened_interest_data.emplace_back(std::move(data));
	460	+ flattened_idx_to_batch_idx[flattened_idx++] = n;
	461	+ }
	462	+ }
	463	+ ++taskId_iter;
	464	+ }
	465	+
	466	+
	467	+ /* 2. inference. */
	468	+ int n_input_image = flattened_imgs.size();
	469	+ fight_det_result model_results[n_input_image];
	470	+ {
	471	+ int steps = (n_input_image + MAX_BATCH - 1) / MAX_BATCH;
	472	+
	473	+ for (int step = 0; step < steps; ++step)
	474	+ {
	475	+ int offset = step * MAX_BATCH;
	476	+ int batch_size = (step == steps - 1) ? n_input_image - offset : MAX_BATCH;
	477	+ fight_det_process_batch(tools_, flattened_imgs.data() + offset, batch_size, model_results + offset);
	478	+ }
	479	+ }
	480	+
	481	+
	482	+ /* 3. postprocess. */
	483	+ {
	484	+ /* a. review to 2d format. */
	485	+ std::unordered_map<taskid_t, std::vector<fight_det_result>> taskid_to_sdk_result;
	486	+ {
	487	+ for (int n = 0; n < n_input_image; ++n)
	488	+ taskid_to_sdk_result[flattened_interest_data[n].taskId].emplace_back(std::move(model_results[n]));
	489	+ }
	490	+
	491	+ /* b. post process */
	492	+ int n = 0;
	493	+ for (auto iter = taskIds.begin(); iter != taskIds.end(); ++iter)
	494	+ {
	495	+ result_data_t result_data;
	496	+ auto &taskId = *iter;
	497	+
	498	+ auto fall_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fall_algor_type_);
	499	+ auto fall_algor_param = (fall_algor_param_wrap != nullptr) ? ((fall_algor_param_type)fall_algor_param_wrap->algor_param) : nullptr;
	500	+
	501	+ auto fight_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fight_algor_type_);
	502	+ auto fight_algor_param = (fight_algor_param_wrap != nullptr) ? ((fight_algor_param_type)fight_algor_param_wrap->algor_param) : nullptr;
	503	+
	504	+ auto &model_result = taskid_to_sdk_result[taskId];
	505	+ const unsigned model_result_size = model_result.size();
	506	+
	507	+ bool has_fight_pair[model_result_size];
	508	+ memset(has_fight_pair, false, model_result_size * sizeof(bool));
	509	+
	510	+ for (int i = 0; i < model_result_size; ++i, ++n)
	511	+ {
	512	+ const sy_img& src_img = det_input_images[flattened_idx_to_batch_idx[n]];
	513	+ auto &cropped_img = flattened_imgs[n]; // croped image.
	514	+ auto &preprocessed_data = flattened_interest_data[n];
	515	+ auto &fight_result_iter = model_result[i];
	516	+ for (int j = 0; j < FIGHT_MODEL_NUM; ++j)
	517	+ {
	518	+ auto &res = fight_result_iter.fight_infos[j];
	519	+
	520	+ if (fight_algor_param != nullptr)
	521	+ if (!has_fight_pair[i])
	522	+ for (int z = i + 1; z < model_result_size; ++z)
	523	+ if (!has_fight_pair[z] && (res.fight_score > fight_algor_param->threshold \|\|
	524	+ model_result[z].fight_infos[j].fight_score > fight_algor_param->threshold))
	525	+ if (iou(preprocessed_data.box, flattened_interest_data[n + (z - i)].box) > fight_algor_param->iou_threshold)
	526	+ {
	527	+ // std::printf("[Debug] fight Pair ");
	528	+ has_fight_pair[i] = has_fight_pair[z] = true;
	529	+
	530	+ auto &preprocessed_data2 = flattened_interest_data[n + (z - i)];
	531	+ result_fight_data_t data;
	532	+ {
	533	+ data.taskid = taskId;
	534	+ data.objectids.insert(preprocessed_data.objId);
	535	+ data.objectids.insert(preprocessed_data2.objId);
	536	+ data.box = union_box(preprocessed_data.box, preprocessed_data2.box);
	537	+ // data.box.score = (res.fight_score + model_result[z].fight_infos[j].fight_score) * 0.5;
	538	+ data.box.score = std::max(res.fight_score, model_result[z].fight_infos[j].fight_score);
	539	+
	540	+ // std::printf("\tltrb is [%d %d %d %d] wh of src img is [%d %d]", data.box.left, data.box.top, data.box.right, data.box.bottom, src_img.w_, src_img.h_);
	541	+
	542	+ sy_img img;
	543	+ {
	544	+ img.c_ = src_img.c_;
	545	+ img.w_ = data.box.width();
	546	+ img.h_ = data.box.height();
	547	+ }
	548	+ cudaError_t cuda_status;
	549	+ const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char);
	550	+ if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void**)&img.data_, nbytes)))
	551	+ {
	552	+ LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ", cudaGetErrorString(cuda_status), nbytes, nbytes / (1024 * 1024));
	553	+ continue;
	554	+ }
	555	+
	556	+ if (CUDA_SUCCESS != ( cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, data.box.left, data.box.top, img.w_, img.h_)))
	557	+ {
	558	+ LOG_ERROR("Crop image GPU failed error is {} wh is [{} {}] ltrb is [{} {} {} {}]",
	559	+ cudaGetErrorString(cuda_status), src_img.w_, src_img.h_, data.box.left, data.box.top, data.box.right, data.box.bottom);
	560	+ CHECK(cudaFree(img.data_));
	561	+ continue;
	562	+ }
	563	+ data.roi_img = img;
	564	+ data.ori_img = src_img;
	565	+ data.id = gid_++;
	566	+ }
	567	+ result_data.fight_data.emplace_back(std::move(data));
	568	+ }
	569	+
	570	+ if (fall_algor_param != nullptr)
	571	+ if (res.fall_score > fall_algor_param->threshold)
	572	+ {
	573	+ if ((float)(src_img.h_ - preprocessed_data.box.bottom) > (float)(src_img.h_ * 0.05))
	574	+ {
	575	+ result_fall_data_t data;
	576	+ {
	577	+ data.box = preprocessed_data.box;
	578	+ data.objectid = preprocessed_data.objId;
	579	+ data.taskid = taskId;
	580	+ data.roi_img = cropped_img;
	581	+ data.ori_img = src_img;
	582	+ data.id = gid_++;
	583	+ }
	584	+ result_data.fall_data.emplace_back(std::move(data));
	585	+ goto _continue;
	586	+ }
	587	+ }
	588	+
	589	+
	590	+ }
	591	+ cudaFree(cropped_img.data_);
	592	+ _continue:
	593	+ {
	594	+
	595	+ }
	596	+ }
	597	+ results.emplace_back(std::move(result_data));
	598	+ }
	599	+ }
	600	+ return true;
	601	+ } // func end
	602	+
351	603	} // namespace fight_fall_cls
352	604	} // namespace ai_engine_module
353	605
...	...

tsl_aiplatform/ai_engine_module/fight_fall_cls.hpp

View file @28ace27

...	...	@@ -107,6 +107,9 @@ namespace ai_engine_module
107	107	bool process_mstreams(const std::set<taskid_t> &taskIds, const sy_img *det_input_images, const std::vector<onelevel_det_result> &det_result,
108	108	std::vector<result_data_t> &results);
109	109
	110	+ bool process_mstreams2(const std::vector<taskid_t> &taskIds, const sy_img *det_input_images, const std::vector<onelevel_det_result> &det_result,
	111	+ std::vector<result_data_t> &results);
	112	+
110	113	FightfallCls(const FightfallCls&) = delete;
111	114	FightfallCls& operator=(const FightfallCls&) = delete;
112	115
...	...

tsl_aiplatform/ai_platform/MultiSourceProcess.cpp

View file @28ace27

...	...	@@ -249,27 +249,46 @@ bool CMultiSourceProcess::add_task_operation(task_param _cur_task_param){
249	249	// pDecManager->setDecKeyframe(config.name, true); // 只对关键帧解码
250	250	pDecManager->startDecodeByName(config.name);
251	251
	252	+ const char* task_id = _cur_task_param.task_id;
	253	+
	254	+ // 保存新添加任务的配置参数
	255	+ m_task_param_manager->add_task_param(task_id, _cur_task_param);
	256	+
252	257	int input_image_width = 0, input_image_height = 0;
253	258	pDecManager->getResolution(config.name, input_image_width, input_image_height);
254	259
	260	+#ifdef WITH_SECOND_PROCESS
	261	+ /* 如果开启了行人机动车非法闯入功能生成闯入区域mask */
	262	+ auto new_task_algor_param = m_task_param_manager->get_task_other_param(task_id);
255	263
256		- const char *task_id = _cur_task_param.task_id;
	264	+ if (new_task_algor_param->find(algorithm_type_t::PEDESTRIAN_TRESPASS) != new_task_algor_param->end()) {
	265	+ pedestrian_vehicle_trespass_.pedestrianvehicletrespass_init_region(
	266	+ task_id, algorithm_type_t::PEDESTRIAN_TRESPASS, input_image_width, input_image_height);
	267	+ }
257	268
258		- m_task_param_manager->add_task_param(task_id, _cur_task_param);
	269	+ if (new_task_algor_param->find(algorithm_type_t::VEHICLE_TRESPASS) != new_task_algor_param->end()) {
	270	+ pedestrian_vehicle_trespass_.pedestrianvehicletrespass_init_region(
	271	+ task_id, algorithm_type_t::VEHICLE_TRESPASS, input_image_width, input_image_height);
	272	+ }
	273	+#endif
	274	+
	275	+ ((save_snapshot_reprocessing *)m_save_snapshot_reprocessing)->add_newtask(task_id);
259	276
260		-// // 人车物跟踪
261		-// if (task_has_vpt_algor(task_id))
262		-// AddTaskTracker(VPT_Handle_, cur_task.first);
	277	+ // 人车物跟踪
	278	+ if (task_has_vpt_algor(task_id))
	279	+ AddTaskTracker(VPT_Handle_, task_id);
263	280
264		-// // 人脸跟踪
265		-// #ifdef WITH_FACE_DET_SS
266		-// if (task_has_face_algor(task_id))
267		-// m_face_det_ai_engine.operator_tracker(task_id, ADDTASK, SKIP_FRAME); // 跳帧数暂时写死
268		-// #endif
	281	+ // 人脸跟踪
	282	+#ifdef WITH_FACE_DET_SS
	283	+ if (task_has_face_algor(task_id))
	284	+ m_face_det_ai_engine.operator_tracker(task_id, ADDTASK, SKIP_FRAME); // 跳帧数暂时写死
	285	+#endif
269	286
	287	+ // 启动算法处理线程
270	288	startProcessByGpuid(_cur_task_param.gpu_id_);
271	289	}
272	290
	291	+// 启动算法处理线程
273	292	void CMultiSourceProcess::startProcessByGpuid(const string gpuid){
274	293
275	294	struct ThreadArg{
...	...	@@ -298,7 +317,7 @@ void CMultiSourceProcess::startProcessByGpuid(const string gpuid){
298	317	}
299	318
300	319	void CMultiSourceProcess::post_decode_thread(task_param _cur_task_param, AVFrame * gpuFrame){
301		- if (gpuFrame->format == AV_PIX_FMT_CUDA){
	320	+ if (gpuFrame->format == AV_PIX_FMT_CUDA){
302	321	GpuRgbMemory* gpuMem = new GpuRgbMemory(3, gpuFrame->width, gpuFrame->height, _cur_task_param.task_id, _cur_task_param.gpu_id_ , true);
303	322
304	323	cudaSetDevice(atoi(_cur_task_param.gpu_id_));
...	...	@@ -530,16 +549,16 @@ void CMultiSourceProcess::algorthim_pedestrian_safety(vector<string>& vpt_intere
530	549	}
531	550
532	551	// 逆行&非法闯入算法模块
533		-void CMultiSourceProcess::algorthim_retrograde_trespass(vector<string>& vpt_interest_task_id, vector<sy_img>& vpt_interest_imgs, vector<onelevel_det_result>& vptResult
534		- ,vector<vector<int>>& deleteObjectID){
	552	+void CMultiSourceProcess::algorthim_retrograde_trespass(vector<string>& vpt_interest_task_id, vector<sy_img>& vpt_interest_imgs
	553	+ , vector<onelevel_det_result>& vptResult ,vector<vector<int>>& deleteObjectID){
535	554	#ifdef WITH_SECOND_PROCESS
536	555	vector<string> interest_task_id;
537		- decltype(vptResult) interest_vpt_result;
	556	+ vector<onelevel_det_result> interest_vpt_result;
538	557	vector<sy_img> interest_imgs(0);
539	558
540	559	vector<string> trespass_interest_task_id;
541		- decltype(vptResult) trespass_interest_vpt_result;
542		- decltype(deleteObjectID) trespass_interest_deleteobjs;
	560	+ vector<onelevel_det_result> trespass_interest_vpt_result;
	561	+ vector<vector<int>> trespass_interest_deleteobjs;
543	562	vector<sy_img> trespass_interest_imgs(0);
544	563
545	564	int _idx = 0;
...	...	@@ -847,7 +866,7 @@ void CMultiSourceProcess::algorithm_fight_fall(vector<string>& vpt_interest_task
847	866	/* b. process result. */
848	867	if (!interest_imgs.empty()) {
849	868	std::vector<ai_engine_module::fight_fall_cls::result_data_t> results;
850		- fight_fall_cls_.process_mstreams(interest_task_id, interest_imgs.data(), interest_vpt_result, results);
	869	+ fight_fall_cls_.process_mstreams2(interest_task_id, interest_imgs.data(), interest_vpt_result, results);
851	870	for (auto &result : results) {
852	871	for (auto &res : result.fight_data) {
853	872	auto &taskId = res.taskid;
...	...	@@ -878,7 +897,7 @@ void CMultiSourceProcess::algorithm_takeaway_member_cls(vector<string>& vpt_inte
878	897	#ifdef WITH_SECOND_PROCESS
879	898	/* find takeaway member classification taskId. */
880	899	vector<string> interest_task_id;
881		- decltype(vptResult) interest_vpt_result;
	900	+ vector<onelevel_det_result> interest_vpt_result;
882	901	vector<sy_img> interest_imgs(0);
883	902
884	903	int _idx = 0;
...	...	@@ -887,7 +906,7 @@ void CMultiSourceProcess::algorithm_takeaway_member_cls(vector<string>& vpt_inte
887	906	{
888	907	const auto &task_id = *_task_id_iter;
889	908
890		- auto algor_map = pThreadParam->m_task_param_manager->get_task_other_param(task_id);
	909	+ auto algor_map = m_task_param_manager->get_task_other_param(task_id);
891	910	if (algor_map->find(algorithm_type_t::TAKEAWAY_MEMBER_CLASSIFICATION) == algor_map->end())
892	911	continue;
893	912
...	...	@@ -907,7 +926,7 @@ int CMultiSourceProcess::everyframe_process(vector<string> &task_in_play_id, sy_
907	926	vector<onelevel_det_result> &ol_det_result) {
908	927	#ifdef WITH_SECOND_PROCESS
909	928	/* 人数聚集算法功能每帧都会获取算法结果并返回 */
910		- auto results = m_human_gather_statistics::human_gather_statistics_process2(task_in_play_id, images, ol_det_result);
	929	+ auto results = m_human_gather_statistics.human_gather_statistics_process2(task_in_play_id, images, ol_det_result);
911	930
912	931	for (auto &result : results) {
913	932	#ifdef POST_USE_RABBITMQ
...	...

tsl_aiplatform/ai_platform/MultiSourceProcess.h

View file @28ace27

...	...	@@ -210,10 +210,10 @@ private:
210	210	int endframe_obj_process(const OBJ_KEY &obj_key, algorithm_type_t algor_type);
211	211	/* 实现快照保存功能（还未真正保存将显存图片cp到内存
212	212	* 直接保存本地或者存入缓存队列异步保存，保存方式看需求，报警类需要同步保存报警，分析类可异步保存后返回）*/
213		- bool save_snapshot_process(const OBJ_KEY &obj_key, const algorithm_type_t &algorithm_type,
214		- const sy_img &ori_img, const sy_img &roi_img, const long long id,
215		- const std::string &json_str, bool enable_async,
216		- const bool ori_img_is_in_gpu, const bool roi_img_is_in_gpu);
	213	+ bool save_snapshot_process(const OBJ_KEY &obj_key, const algorithm_type_t &algorithm_type, const sy_img &ori_img,
	214	+ const sy_img &roi_img, const long long id, const std::string &json_str,
	215	+ bool enable_async = true, const bool ori_img_is_in_gpu = true,
	216	+ const bool roi_img_is_in_gpu = true);
217	217
218	218	private:
219	219	map<string, pthread_t*> gpuProcessthreadMap;
...	...