/* * @Author: yangzilong * @Date: 2021-12-02 14:52:24 * @Last Modified by: yangzilong * @Last Modified time: Do not edit * @Email: yangzilong@objecteye.com * @Description: */ #include #include "sy_common.h" #include "./fight_fall_cls.hpp" #include "../reprocessing_module/CropImg.h" namespace ai_engine_module { namespace fight_fall_cls { long long FightfallCls::gid_ = 0; algorithm_type_t FightfallCls::fall_algor_type_ = algorithm_type_t::PEDESTRIAN_FALL; algorithm_type_t FightfallCls::fight_algor_type_ = algorithm_type_t::PEDESTRIAN_FIGHT; FightfallCls::FightfallCls() : task_param_manager_(nullptr) { } FightfallCls::~FightfallCls() { fight_det_release(&tools_); if (!tools_) { delete tools_; tools_ = nullptr; } } bool FightfallCls::init(int gpu_id, char *trt_serialize_file) { init_ = false; fight_det_param param; { param.mode = gpu_id >= 0 ? DEVICE_GPU : DEVICE_CPU; param.gpuid = gpu_id; param.score_thresld = 0.8; param.auth_license = "sy_tsl_aiplatform_sdk_2021"; // TODO: param.max_batch = MAX_BATCH; param.serialize_file = "./serialize_file/fight_fall"; } int status; if (!(init_ = (0 == (status = fight_det_init(&tools_, param))))) LOG_ERROR("Init FightFellClsSdk failed error code is {}", status); else if (!task_param_manager_) task_param_manager_ = task_param_manager::getInstance(); return init_; } bool FightfallCls::check_initied() { if (!init_) LOG_ERROR("[FightfallCls] call init function please."); return init_; } float iou(const box_t &box1, const box_t &box2) { float insect_top = max(box1.top, box2.top); float insect_left = max(box1.left, box2.left); float insect_right = min(box1.right, box2.right); float insect_bottom = min(box1.bottom, box2.bottom); if (insect_right <= insect_left || insect_bottom <= insect_top) return 0.0f; float insect_area = (insect_right - insect_left) * (insect_bottom - insect_top); float union_area = (box1.right - box1.left) * (box1.bottom - box1.top) + (box2.right - box2.left) * (box2.bottom - box2.top) - insect_area; return insect_area / union_area; } box_t union_box(const box_t &box1, const box_t &box2) { box_t box; { box.top = std::min(box1.top, box2.top); box.left = std::min(box1.left, box2.left); box.right = std::max(box1.right, box2.right); box.bottom = std::max(box1.bottom, box2.bottom); } return box; } bool FightfallCls::process_mstreams(const std::set &taskIds, const sy_img *det_input_images, const std::vector &det_results, std::vector &results) { if (!check_initied()) return false; if (det_results.empty()) { LOG_ERROR("[FightfallCls] call init function please."); LOG_ERROR("detection result is empty."); return false; } int n_images = det_results.size(); // or n_stream unsigned flattened_idx = 0; std::map flattened_idx_to_batch_idx; /* 1. Crop & keep some interest class. */ /* 1. preprocess. */ auto taskId_iter = taskIds.begin(); std::vector flattened_imgs(0); std::vector flattened_interest_data(0); // for (int n = 0; n < n_images; ++n) { int n_interest_obj = 0; const sy_img &src_img = det_input_images[n]; auto &boxes_of_one_image = det_results[n].obj; for (int i = 0; i < det_results[n].obj_count; ++i) { auto &box = boxes_of_one_image[i]; if (static_cast(box.index) == det_class_label_t::HUMAN) { auto &taskId = *taskId_iter; input_data_wrap_t data; int top = std::max(int(box.top - (IMAGE_CROP_EXPAND_RATIO * box.top)), 0); int left = std::max(int(box.left - (IMAGE_CROP_EXPAND_RATIO * box.left)), 0); int right = std::min(int(box.right + (IMAGE_CROP_EXPAND_RATIO * box.right)), src_img.w_); int bottom = std::min(int(box.bottom + (IMAGE_CROP_EXPAND_RATIO * box.bottom)), src_img.h_); int width = right - left; int height = bottom - top; auto fall_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fall_algor_type_); auto fall_algor_param = (fall_algor_param_wrap != nullptr) ? ((fall_algor_param_type)fall_algor_param_wrap->algor_param) : nullptr; auto fall_basic_param = (fall_algor_param_wrap != nullptr) ? (fall_algor_param_wrap->basic_param) : nullptr; auto fight_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fight_algor_type_); auto fight_algor_param = (fight_algor_param_wrap != nullptr) ? ((fight_algor_param_type)fight_algor_param_wrap->algor_param) : nullptr; auto fight_basic_param = (fight_algor_param_wrap != nullptr) ? (fight_algor_param_wrap->basic_param) : nullptr; auto minimum_width = std::min((fall_algor_param == nullptr ? DEFAULT_MIN_WIDTH : fall_algor_param->pedestrian_min_width), (fight_algor_param == nullptr ? DEFAULT_MIN_WIDTH : fight_algor_param->pedestrian_min_width)); auto minimum_height = std::min((fall_algor_param == nullptr ? DEFAULT_MIN_HEIGHT : fall_algor_param->pedestrian_min_height), (fight_algor_param == nullptr ? DEFAULT_MIN_HEIGHT : fight_algor_param->pedestrian_min_height)); auto minimum_threshold = std::min((fall_algor_param == nullptr ? DEFAULT_PTHRESHOLD : fall_algor_param->threshold), (fight_algor_param == nullptr ? DEFAULT_PTHRESHOLD : fight_algor_param->threshold)); sy_rect intel_rect; if (fight_basic_param == nullptr) { if (fall_basic_param != nullptr) intel_rect = fall_basic_param->algor_valid_rect; } else { intel_rect = fight_basic_param->algor_valid_rect; } if ((width < minimum_width || height < minimum_height || box.confidence < minimum_threshold) || !snapshot_legal_inarea(intel_rect, left, top, right, bottom)) continue; data.box.top = top; data.box.left = left; data.box.right = right; data.box.bottom = bottom; data.box.score = box.confidence; data.taskId = taskId; data.objId = box.id; sy_img img; img.w_ = width; img.h_ = height; img.c_ = src_img.c_; cudaError_t cuda_status; const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char); if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void**)&img.data_, nbytes))) { LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ", cudaGetErrorString(cuda_status), nbytes, nbytes / (1024 * 1024)); continue; } if (CUDA_SUCCESS != ( cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, left, top, width, height))) { LOG_ERROR("Crop image GPU failed error is %s wh is [{}, {}] ltrb is [{} {} {} {}]", cudaGetErrorString(cuda_status), src_img.w_, src_img.h_, data.box.left, data.box.top, data.box.right, data.box.bottom); CHECK(cudaFree(img.data_)); continue; } flattened_imgs.emplace_back(std::move(img)); flattened_interest_data.emplace_back(std::move(data)); flattened_idx_to_batch_idx[flattened_idx++] = n; } } ++taskId_iter; } /* 2. inference. */ int n_input_image = flattened_imgs.size(); fight_det_result model_results[n_input_image]; { int steps = (n_input_image + MAX_BATCH - 1) / MAX_BATCH; for (int step = 0; step < steps; ++step) { int offset = step * MAX_BATCH; int batch_size = (step == steps - 1) ? n_input_image - offset : MAX_BATCH; fight_det_process_batch(tools_, flattened_imgs.data() + offset, batch_size, model_results + offset); } } /* 3. postprocess. */ { /* a. review to 2d format. */ std::unordered_map> taskid_to_sdk_result; { for (int n = 0; n < n_input_image; ++n) taskid_to_sdk_result[flattened_interest_data[n].taskId].emplace_back(std::move(model_results[n])); } /* b. post process */ int n = 0; for (auto iter = taskIds.begin(); iter != taskIds.end(); ++iter) { result_data_t result_data; auto &taskId = *iter; auto fall_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fall_algor_type_); auto fall_algor_param = (fall_algor_param_wrap != nullptr) ? ((fall_algor_param_type)fall_algor_param_wrap->algor_param) : nullptr; auto fight_algor_param_wrap = task_param_manager_->get_task_other_param(taskId, this->fight_algor_type_); auto fight_algor_param = (fight_algor_param_wrap != nullptr) ? ((fight_algor_param_type)fight_algor_param_wrap->algor_param) : nullptr; auto &model_result = taskid_to_sdk_result[taskId]; const unsigned model_result_size = model_result.size(); bool has_fight_pair[model_result_size]; memset(has_fight_pair, false, model_result_size * sizeof(bool)); for (int i = 0; i < model_result_size; ++i, ++n) { const sy_img& src_img = det_input_images[flattened_idx_to_batch_idx[n]]; auto &cropped_img = flattened_imgs[n]; // croped image. auto &preprocessed_data = flattened_interest_data[n]; auto &fight_result_iter = model_result[i]; for (int j = 0; j < FIGHT_MODEL_NUM; ++j) { auto &res = fight_result_iter.fight_infos[j]; if (fight_algor_param != nullptr) if (!has_fight_pair[i]) for (int z = i + 1; z < model_result_size; ++z) if (!has_fight_pair[z] && (res.fight_score > fight_algor_param->threshold || model_result[z].fight_infos[j].fight_score > fight_algor_param->threshold)) if (iou(preprocessed_data.box, flattened_interest_data[n + (z - i)].box) > fight_algor_param->iou_threshold) { // std::printf("[Debug] fight Pair "); has_fight_pair[i] = has_fight_pair[z] = true; auto &preprocessed_data2 = flattened_interest_data[n + (z - i)]; result_fight_data_t data; { data.taskid = taskId; data.objectids.insert(preprocessed_data.objId); data.objectids.insert(preprocessed_data2.objId); data.box = union_box(preprocessed_data.box, preprocessed_data2.box); // data.box.score = (res.fight_score + model_result[z].fight_infos[j].fight_score) * 0.5; data.box.score = std::max(res.fight_score, model_result[z].fight_infos[j].fight_score); // std::printf("\tltrb is [%d %d %d %d] wh of src img is [%d %d]", data.box.left, data.box.top, data.box.right, data.box.bottom, src_img.w_, src_img.h_); sy_img img; { img.c_ = src_img.c_; img.w_ = data.box.width(); img.h_ = data.box.height(); } cudaError_t cuda_status; const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char); if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void**)&img.data_, nbytes))) { LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ", cudaGetErrorString(cuda_status), nbytes, nbytes / (1024 * 1024)); continue; } if (CUDA_SUCCESS != ( cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, data.box.left, data.box.top, img.w_, img.h_))) { LOG_ERROR("Crop image GPU failed error is {} wh is [{} {}] ltrb is [{} {} {} {}]", cudaGetErrorString(cuda_status), src_img.w_, src_img.h_, data.box.left, data.box.top, data.box.right, data.box.bottom); CHECK(cudaFree(img.data_)); continue; } data.roi_img = img; data.ori_img = src_img; data.id = gid_++; } result_data.fight_data.emplace_back(std::move(data)); } if (fall_algor_param != nullptr) if (res.fall_score > fall_algor_param->threshold) { if ((float)(src_img.h_ - preprocessed_data.box.bottom) > (float)(src_img.h_ * 0.05)) { result_fall_data_t data; { data.box = preprocessed_data.box; data.objectid = preprocessed_data.objId; data.taskid = taskId; data.roi_img = cropped_img; data.ori_img = src_img; data.id = gid_++; } result_data.fall_data.emplace_back(std::move(data)); goto _continue; } } } cudaFree(cropped_img.data_); _continue: { } } results.emplace_back(std::move(result_data)); } } return true; } // func end } // namespace fight_fall_cls } // namespace ai_engine_module