/* * File: pedestrian_safety_det.cpp * Created Date: Tuesday February 15th 2022 * Author: yangzilong (yangzilong@objecteye.com) * Description: * ----- * Last Modified: Tuesday, 15th February 2022 6:19:32 pm * Modified By: yangzilong (yangzilong@objecteye.com>) * ----- * Copyright 2022 */ #include "./pedestrian_safety_det.hpp" #include "../reprocessing_module/CropImg.h" namespace ai_engine_module { namespace pedestrian_safety_det { #define INPUT_WIDTH 320 #define INPUT_HEIGHT 320 #define INPUT_CHANNEL 3 static std::set algor_type_list_ = { algorithm_type_t::SMOKING_DET, algorithm_type_t::CALL_PHONE_DET, algorithm_type_t::NO_REFLECTIVE_CLOTHING, algorithm_type_t::NO_SAFETY_HELMET, }; static std::set not_semantic_algor_type_list_ = { algorithm_type_t::NO_REFLECTIVE_CLOTHING, algorithm_type_t::NO_SAFETY_HELMET, }; inline bool is_valid_label(const label_t &label) { return ((label == label_t::helmet) || (label == label_t::phone) || (label == label_t::smoke) || (label == label_t::vest)); } inline bool is_support_algorithm_type(const algorithm_type_t &algor_type) { return algor_type_list_.count(algor_type) > 0; } // ############################################################ // // ! Auxiliary Function ! // // ############################################################ // std::set task_id_to_algorithm_type_seq(const task_id_t &task_id, task_param_manager *const task_param) { std::set seq; auto &&algor_map = task_param->get_task_other_param(task_id); if (algor_map) { // LOG_TRACE("task id is {} size algor type {}", task_id, algor_map->size()); for (auto iter = algor_map->begin(); iter != algor_map->end(); ++iter) { if (algor_type_list_.count(iter->first) > 0) seq.emplace(iter->first); } } return seq; // N(RVO) } pedestrian_safety_det::label_t algor_type_to_label(const algorithm_type_t &algor_type) { using namespace pedestrian_safety_det; switch (algor_type) { case algorithm_type_t::NO_SAFETY_HELMET: return label_t::helmet; case algorithm_type_t::CALL_PHONE_DET: return label_t::phone; case algorithm_type_t::SMOKING_DET: return label_t::smoke; case algorithm_type_t::NO_REFLECTIVE_CLOTHING: return label_t::vest; // default: return label_t::PLACEHOLDER; } } algorithm_type_t label_to_algor_type(const label_t &label) { using namespace pedestrian_safety_det; switch (label) { case label_t::helmet: return algorithm_type_t::NO_SAFETY_HELMET; case label_t::phone: return algorithm_type_t::CALL_PHONE_DET; case label_t::smoke: return algorithm_type_t::SMOKING_DET; case label_t::vest: return algorithm_type_t::NO_REFLECTIVE_CLOTHING; default: return algorithm_type_t::PLACEHOLDER; } } size_t PedestrianSafetyDetector::get_number_of_results() { size_t res{0}; for (auto iter = id_to_results_.cbegin(); iter != id_to_results_.cend(); ++iter) { res += iter->second.size(); } return res; } bool is_valid_box(const int top, const int left, const int right, const int bottom, const float score, const algorithm_type_t &algor_type, const task_param_manager::algo_param_type_t_ *params_ptr = nullptr) { if (!params_ptr) return false; if (!snapshot_legal_inarea(params_ptr->basic_param->algor_valid_rect, left, top, right, bottom)) return false; if (params_ptr->algor_param == nullptr) return false; const unsigned width = right - left; const unsigned height = bottom - top; if (width == 0 || height == 0) return false; //! TODO: use switch to replace. using data_t = algor_config_param_pedestrian_safety_detector_basic; data_t *algor_params_ptr = (data_t *) (params_ptr->algor_param); if (score < algor_params_ptr->pedestrian_confidence_threshold || width < algor_params_ptr->pedestrian_min_width || height < algor_params_ptr->pedestrian_min_height) return false; return true; } bool is_valid_box(const float score, const label_t &label, const algorithm_type_t &algor_type, const task_param_manager::algo_param_type_t_ *params_ptr = nullptr) { if (!params_ptr) return false; if (params_ptr->algor_param == nullptr) return false; //! TODO: use switch to replace. using data_t = algor_config_param_pedestrian_safety_detector_basic; data_t *algor_params_ptr = (data_t *) (params_ptr->algor_param); if (label != algor_type_to_label(algor_type) || score < algor_params_ptr->conf_threshold) return false; return true; } bool update_and_get_mn_status(const obj_key_t &obj_key, const label_t &label, const bool do_erase, obj_key_to_mn_strategy_t &obj_key_to_mn_strategy, const task_param_manager::algo_param_type_t_ *params_ptr = nullptr) { if (params_ptr == nullptr) { LOG_ERROR(" params_ptr is null."); return false; } if (params_ptr->algor_param == nullptr) { LOG_ERROR(" params -> algor_param is null."); return false; } //! TODO: use switch to replace. using data_t = algor_config_param_pedestrian_safety_detector_basic; data_t *algor_params_ptr = (data_t *) (params_ptr->algor_param); using label_t = pedestrian_safety_det::label_t; auto &mn_strategy = obj_key_to_mn_strategy[obj_key]; mn_strategy.set_target_m(algor_params_ptr->m); mn_strategy.set_target_n(algor_params_ptr->n); if (label != label_t::PLACEHOLDER) { label_t target_label = algor_type_to_label(obj_key.algor_type); if (label_t::PLACEHOLDER == target_label) return false; return mn_strategy.update_and_check(label, target_label, /*do_reset=*/do_erase); } bool ret = mn_strategy.update_and_check(/*do_erase=*/do_erase); return ret; } // ############################################################ // // ! Class Member ! // // ############################################################ // PedestrianSafetyDetector::PedestrianSafetyDetector() : task_param_manager_(nullptr), handle_(nullptr), batch_size_(0), initied_(false) { } PedestrianSafetyDetector::~PedestrianSafetyDetector() { if (handle_ != nullptr) { hat_callsmoke_vestuniform_det_release(&handle_); if (!handle_) { delete handle_; handle_ = nullptr; } } } bool PedestrianSafetyDetector::init(const hat_callsmoke_vestuniform_det_param &initied_params) { int status = 0; if (!(initied_ = (0 == (status = hat_callsmoke_vestuniform_det_init(&handle_, initied_params))))) { LOG_ERROR("Init PedestrianSafetyDetectorSdk failed error code is {}", status); return false; } batch_size_ = initied_params.max_batch; if (!task_param_manager_) task_param_manager_ = task_param_manager::getInstance(); return initied_; } bool PedestrianSafetyDetector::is_support(const algorithm_type_t &algor_type) const noexcept { return algor_type_list_.count(algor_type) > 0; } bool PedestrianSafetyDetector::check_initied() const { if (!initied_) LOG_ERROR("[%s:%d] call init function please.", __FILE__, __LINE__); return initied_; } std::shared_ptr PedestrianSafetyDetector::get_results_by_id(const unique_obj_id_t &id, bool do_erase) { auto it = id_to_results_.find(id); if (it == id_to_results_.end()) return nullptr; // std::shared_ptr res(&(it->second)); std::shared_ptr res = std::make_shared(std::move(it->second)); if (do_erase) id_to_results_.erase(id); return res; } bool PedestrianSafetyDetector::update_mstreams(const std::set &tasks_id, const sy_img *det_input_images, const std::vector &det_results) { if (!check_initied()) return false; if (tasks_id.empty()) { LOG_DEBUG("task_id is empty."); return false; } struct stream_idx_and_algor_seq_t { unsigned stream_idx; std::set algors; }; // LOG_TRACE("number of tasks_id is {}", tasks_id.size()); unsigned flattened_idx = 0; //! 记录每个box对应的算法以及流id. std::map flattened_idx_to_algor_seq; /* 1. Crop & keep some interest class. */ unsigned stream_idx = 0; std::vector flattened_imgs(0); std::vector flattened_interest_data(0); // for (auto iter = tasks_id.begin(); iter != tasks_id.end(); ++iter, ++stream_idx) { task_id_t task_id = *iter; auto &src_img = det_input_images[stream_idx]; auto &boxes_of_one_image = det_results[stream_idx].obj; for (int i = 0; i < det_results[stream_idx].obj_count; ++i) //! loop per box. { auto &box = boxes_of_one_image[i]; if (static_cast(box.index) != det_class_label_t::HUMAN) continue; //! get cropped box. input_data_wrap_t data; int top = std::max(int(box.top - (IMAGE_CROP_EXPAND_RATIO * box.top)), 0); int left = std::max(int(box.left - (IMAGE_CROP_EXPAND_RATIO * box.left)), 0); int right = std::min(int(box.right + (IMAGE_CROP_EXPAND_RATIO * box.right)), src_img.w_); int bottom = std::min(int(box.bottom + (IMAGE_CROP_EXPAND_RATIO * box.bottom)), src_img.h_); //! loop per algor from set. stream_idx_and_algor_seq_t stream_idx_and_algor_seq{stream_idx, {}}; std::set algorithm_type_seq = task_id_to_algorithm_type_seq(task_id, task_param_manager_); // N(RVO). for (auto algor_iter = algorithm_type_seq.begin(); algor_iter != algorithm_type_seq.end(); ++algor_iter) { const algorithm_type_t algor_type = *algor_iter; auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type); if (!algor_param_wrap) { LOG_ERROR("{} is nullptr when get algor param from task_param", task_id.c_str()); continue; } if (!is_valid_box(top, left, right, bottom, box.confidence, algor_type, algor_param_wrap)) continue; stream_idx_and_algor_seq.algors.emplace(algor_type); } if (stream_idx_and_algor_seq.algors.empty()) continue; int width = right - left; int height = bottom - top; data.box.top = top; data.box.left = left; data.box.right = right; data.box.bottom = bottom; data.taskId = task_id; data.objId = box.id; sy_img img; { img.w_ = width; img.h_ = height; img.c_ = src_img.c_; img.data_ = nullptr; } cudaError_t cuda_status; const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char); if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void **) &img.data_, nbytes))) { LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ", cudaGetErrorString(cuda_status), nbytes, nbytes / (1024 * 1024)); continue; } //! TODO: use long short edge resize to replace. if (CUDA_SUCCESS != (cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, left, top, width, height))) { LOG_ERROR("Crop image GPU failed error is {} wh is [{}, {}] ltrb is [{} {} {} {}]", cudaGetErrorString(cuda_status), src_img.w_, src_img.h_, data.box.left, data.box.top, data.box.right, data.box.bottom); CHECK(cudaFree(img.data_)); continue; } flattened_imgs.emplace_back(std::move(img)); flattened_interest_data.emplace_back(std::move(data)); flattened_idx_to_algor_seq[flattened_idx++] = std::move(stream_idx_and_algor_seq); } } if (flattened_imgs.empty()) return true; // printf("[Ckpt1] number of f_imgs f_data is %lu %lu\n", flattened_imgs.size(), flattened_interest_data.size()); LOG_TRACE("[Ckpt1] number of f_imgs f_data is {} {}", flattened_imgs.size(), flattened_interest_data.size()); /* 2. do inference and get result. */ const unsigned n_input_image = flattened_imgs.size(); LOG_TRACE("input image size {}", n_input_image); hat_callsmoke_vestuniform_det_result model_results[n_input_image]; { const unsigned stride = batch_size_; const unsigned steps = (n_input_image + stride - 1) / stride; for (unsigned step = 0; step < steps; ++step) { const unsigned offset = step * stride; const unsigned batch_size = (step == steps - 1) ? n_input_image - offset : stride; // printf(" Step: [%d] start_idx: %u batch_size: %u\n", step, offset, batch_size); LOG_TRACE(" Step: [{}] start_idx: {} batch_size: {}", step, offset, batch_size); hat_callsmoke_vestuniform_det_batch(handle_, flattened_imgs.data() + offset, batch_size, model_results + offset); } } // printf("[Ckpt2] number of f_imgs f_data is %lu %lu\n", flattened_imgs.size(), flattened_interest_data.size()); LOG_TRACE("[Ckpt2] number of f_imgs f_data is {} {}", flattened_imgs.size(), flattened_interest_data.size()); // return true; /* 3. postprocess. */ { //! loop per box output. for (unsigned n = 0; n < n_input_image; ++n) { auto &det_result = flattened_interest_data[n]; auto &obj_id = det_result.objId; auto &task_id = det_result.taskId; auto &stream_idx_and_algor_seq = flattened_idx_to_algor_seq[n]; auto &steram_idx = stream_idx_and_algor_seq.stream_idx; auto &src_img = det_input_images[steram_idx]; auto &algors = stream_idx_and_algor_seq.algors; auto &model_result = model_results[n]; //! loop per cropped image result. bool is_alarm = false; // LOG_DEBUG("[{}] box count {}", task_id, model_result.obj_count_); //! a. post process. std::map> algor_to_boxes; { for (auto algor_iter = algors.begin(); algor_iter != algors.end(); ++algor_iter) algor_to_boxes[*algor_iter] = std::vector(); } for (unsigned i = 0; i < model_result.obj_count_; ++i) { auto &box = model_result.obj_results_[i]; const label_t label = static_cast(box.obj_index); // LOG_TRACE("task id is {} obj_id {} label {} score {}", task_id, obj_id, label, box.obj_score); if (!is_valid_label(label)) continue; for (auto algor_type_iter = algors.begin(); algor_type_iter != algors.end(); ++algor_type_iter) { const algorithm_type_t algor_type = *algor_type_iter; // LOG_TRACE("task id is {} algor type is {} obj_id {}", task_id, int(algor_type), obj_id); auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type); if (!algor_param_wrap) { LOG_ERROR("{} is nullptr when get algor param from task_param", task_id); continue; } // LOG_TRACE("task id is {} algor type is {} obj_id {} label {} score {}", task_id, int(algor_type), obj_id, label, box.obj_score); if (!is_valid_box(box.obj_score, static_cast(box.obj_index), algor_type, algor_param_wrap)) continue; LOG_TRACE("task id is {} algor type is {} obj_id {}", task_id, int(algor_type), obj_id); box_t c_box; { c_box.cls = box.obj_index; c_box.score = box.obj_score; /* c_box.top = box.obj_rect.top_ + det_result.box.top; c_box.left = box.obj_rect.left_ + det_result.box.left; // c_box.right = c_box.left + box.obj_rect.left_ + box.obj_rect.width_; // c_box.bottom = c_box.top + box.obj_rect.top_ + box.obj_rect.height_; c_box.right = c_box.left + box.obj_rect.width_; // modified byzsh 220803 c_box.bottom = c_box.top + box.obj_rect.height_; // modified byzsh 220803*/ c_box.top = det_result.box.top; c_box.left = det_result.box.left; c_box.right = det_result.box.right; c_box.bottom = det_result.box.bottom; // modified byzsh 220803 此处调整为了行人框，后续可根据需要换回小框 } LOG_TRACE(" box {} ltrb [{} {} {} {}] score {} label {} obj_id {}", det_result.box.id, c_box.left, c_box.top, c_box.right, c_box.bottom, c_box.score, c_box.cls, obj_id); algor_to_boxes[algor_type].emplace_back(std::move(c_box)); // only save max confidence box? } } //! b. alarm strategy. { for (auto iter = algor_to_boxes.begin(); iter != algor_to_boxes.end(); ++iter) { std::vector &boxes = iter->second; const algorithm_type_t algor_type = iter->first; //! filter. if (not_semantic_algor_type_list_.count(algor_type)) { if (boxes.size()) continue; } else { if (boxes.empty()) continue; } // 221009 byzsh记录10条即可------------------------------------------------------------------------------ unique_obj_id_t unique_id_{obj_id, task_id}; if (id_to_results_.find(unique_id_) != id_to_results_.end() && id_to_results_[unique_id_].size() >= 10) continue; //------------------------------------------------------------------------------------------------------ obj_key_t obj_key{obj_id, task_id, algor_type}; LOG_TRACE("================> task_id {} algor_type {} obj_id {}", task_id, int(algor_type), obj_id); auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type); //! if (!update_and_get_mn_status(obj_key, label_t::PLACEHOLDER, /*do_erase=*/false, obj_key_to_mn_strategy_, algor_param_wrap)) continue; LOG_TRACE("22222222222 task_id {} algor_type {} obj_id {}", task_id, int(algor_type), obj_id); result_data_t result_data; { result_data.obj_id = obj_id; result_data.task_id = task_id; } if (not_semantic_algor_type_list_.count(algor_type)) { //! TODO: 使用行人box + 安全帽score?? 如果没有安全帽呢 result_data.box = det_result.box; result_data.box.score = 0.0f; } else { //! save maximum confidence box. auto max_ele = std::max_element(std::begin(boxes), std::end(boxes), [](box_t const &lhs, box_t const &rhs) { return lhs.score < rhs.score; }); result_data.box = *max_ele; // use move segmantic? } #ifdef _USE_SHALLOW_COPY { result.ori_img = src_img; result_data.roi_img = flattened_imgs[n]; } #else { { sy_img img; { img.c_ = src_img.c_; img.h_ = src_img.h_; img.w_ = src_img.w_; img.data_ = nullptr; } const unsigned size = img.c_ * img.h_ * img.w_; img.data_ = new unsigned char[size]; // img.data_ = new std::remove_pointer[size]; CHECK(cudaMemcpy(img.data_, src_img.data_, size * sizeof(unsigned char), cudaMemcpyDeviceToHost)); result_data.ori_img = std::move(img); result_data.ori_img_is_in_gpu = false; } { auto &src_img = flattened_imgs[n]; sy_img img; { img.c_ = src_img.c_; img.h_ = src_img.h_; img.w_ = src_img.w_; img.data_ = nullptr; } const unsigned size = img.c_ * img.h_ * img.w_; img.data_ = new unsigned char[size]; CHECK(cudaMemcpy(img.data_, src_img.data_, size * sizeof(unsigned char), cudaMemcpyDeviceToHost)); result_data.roi_img = std::move(img); // deep copy. // result_data.ori_img_is_in_gpu = false; result_data.roi_img_is_in_gpu = false; //221009 } } #endif is_alarm = true; result_data.algorithm_type_seq.emplace(algor_type); //------------------------------------------------------- // if (result_data.roi_img.data_ != nullptr) { // if (result_data.roi_img_is_in_gpu) { // CHECK(cudaFree((void *)(result_data.roi_img.data_)));} // else { //221007 // LOG_TRACE("PEDESTRIAN free roi cpu memory."); // delete[] result_data.roi_img.data_; // } // result_data.roi_img.data_ = nullptr; // } // if (result_data.ori_img.data_ != nullptr) { // if (result_data.ori_img_is_in_gpu) { // CHECK(cudaFree((void *)(result_data.ori_img.data_)));} // else { //221007 // LOG_TRACE("PEDESTRIAN free ori cpu memory."); // delete[] result_data.ori_img.data_; // } // result_data.ori_img.data_ = nullptr; // } //---------------------------------------------------------- id_to_results_[unique_obj_id_t{obj_id, task_id}].emplace_back(std::move(result_data)); // LOG_TRACE("has alarm task_id {} obj_id {} size {}", task_id, obj_id, id_to_results_[unique_id_].size()); } } } } //! save desc info. for (auto &img: flattened_imgs) { // LOG_TRACE("free gpu memory xxx"); // PRINT_CHECK(cudaFree(img.data_)); if (cudaSuccess != cudaFree(img.data_)) { LOG_ERROR("free image failred."); } void *ptr = nullptr; if (cudaSuccess != cudaFree(ptr)) { LOG_ERROR("free nullptr failed."); } } return true; } bool PedestrianSafetyDetector::update_mstreams2(const std::vector &tasks_id, const sy_img *det_input_images, const std::vector &det_results) { if (!check_initied()) return false; if (tasks_id.empty()) { LOG_DEBUG("task_id is empty."); return false; } struct stream_idx_and_algor_seq_t { unsigned stream_idx; std::set algors; }; // LOG_TRACE("number of tasks_id is {}", tasks_id.size()); unsigned flattened_idx = 0; //! 记录每个box对应的算法以及流id. std::map flattened_idx_to_algor_seq; /* 1. Crop & keep some interest class. */ unsigned stream_idx = 0; std::vector flattened_imgs(0); std::vector flattened_interest_data(0); // for (auto iter = tasks_id.begin(); iter != tasks_id.end(); ++iter, ++stream_idx) { task_id_t task_id = *iter; auto &src_img = det_input_images[stream_idx]; auto &boxes_of_one_image = det_results[stream_idx].obj; for (int i = 0; i < det_results[stream_idx].obj_count; ++i) //! loop per box. { auto &box = boxes_of_one_image[i]; if (static_cast(box.index) != det_class_label_t::HUMAN) continue; //! get cropped box. input_data_wrap_t data; int top = std::max(int(box.top - (IMAGE_CROP_EXPAND_RATIO * box.top)), 0); int left = std::max(int(box.left - (IMAGE_CROP_EXPAND_RATIO * box.left)), 0); int right = std::min(int(box.right + (IMAGE_CROP_EXPAND_RATIO * box.right)), src_img.w_); int bottom = std::min(int(box.bottom + (IMAGE_CROP_EXPAND_RATIO * box.bottom)), src_img.h_); //! loop per algor from set. stream_idx_and_algor_seq_t stream_idx_and_algor_seq{stream_idx, {}}; std::set algorithm_type_seq = task_id_to_algorithm_type_seq(task_id, task_param_manager_); // N(RVO). for (auto algor_iter = algorithm_type_seq.begin(); algor_iter != algorithm_type_seq.end(); ++algor_iter) { const algorithm_type_t algor_type = *algor_iter; auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type); if (!algor_param_wrap) { LOG_ERROR("{} is nullptr when get algor param from task_param", task_id.c_str()); continue; } if (!is_valid_box(top, left, right, bottom, box.confidence, algor_type, algor_param_wrap)) continue; stream_idx_and_algor_seq.algors.emplace(algor_type); } if (stream_idx_and_algor_seq.algors.empty()) continue; int width = right - left; int height = bottom - top; data.box.top = top; data.box.left = left; data.box.right = right; data.box.bottom = bottom; data.taskId = task_id; data.objId = box.id; sy_img img; { img.w_ = width; img.h_ = height; img.c_ = src_img.c_; img.data_ = nullptr; } cudaError_t cuda_status; const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char); if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void **) &img.data_, nbytes))) { LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ", cudaGetErrorString(cuda_status), nbytes, nbytes / (1024 * 1024)); continue; } //! TODO: use long short edge resize to replace. if (CUDA_SUCCESS != (cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, left, top, width, height))) { LOG_ERROR("Crop image GPU failed error is {} wh is [{}, {}] ltrb is [{} {} {} {}]", cudaGetErrorString(cuda_status), src_img.w_, src_img.h_, data.box.left, data.box.top, data.box.right, data.box.bottom); CHECK(cudaFree(img.data_)); continue; } flattened_imgs.emplace_back(std::move(img)); flattened_interest_data.emplace_back(std::move(data)); flattened_idx_to_algor_seq[flattened_idx++] = std::move(stream_idx_and_algor_seq); } } if (flattened_imgs.empty()) return true; // printf("[Ckpt1] number of f_imgs f_data is %lu %lu\n", flattened_imgs.size(), flattened_interest_data.size()); LOG_TRACE("[Ckpt1] number of f_imgs f_data is {} {}", flattened_imgs.size(), flattened_interest_data.size()); /* 2. do inference and get result. */ const unsigned n_input_image = flattened_imgs.size(); LOG_TRACE("input image size {}", n_input_image); hat_callsmoke_vestuniform_det_result model_results[n_input_image]; { const unsigned stride = batch_size_; const unsigned steps = (n_input_image + stride - 1) / stride; for (unsigned step = 0; step < steps; ++step) { const unsigned offset = step * stride; const unsigned batch_size = (step == steps - 1) ? n_input_image - offset : stride; // printf(" Step: [%d] start_idx: %u batch_size: %u\n", step, offset, batch_size); LOG_TRACE(" Step: [{}] start_idx: {} batch_size: {}", step, offset, batch_size); hat_callsmoke_vestuniform_det_batch(handle_, flattened_imgs.data() + offset, batch_size, model_results + offset); } } // printf("[Ckpt2] number of f_imgs f_data is %lu %lu\n", flattened_imgs.size(), flattened_interest_data.size()); LOG_TRACE("[Ckpt2] number of f_imgs f_data is {} {}", flattened_imgs.size(), flattened_interest_data.size()); // return true; /* 3. postprocess. */ { //! loop per box output. for (unsigned n = 0; n < n_input_image; ++n) { auto &det_result = flattened_interest_data[n]; auto &obj_id = det_result.objId; auto &task_id = det_result.taskId; auto &stream_idx_and_algor_seq = flattened_idx_to_algor_seq[n]; auto &steram_idx = stream_idx_and_algor_seq.stream_idx; auto &src_img = det_input_images[steram_idx]; auto &algors = stream_idx_and_algor_seq.algors; auto &model_result = model_results[n]; //! loop per cropped image result. bool is_alarm = false; // LOG_DEBUG("[{}] box count {}", task_id, model_result.obj_count_); //! a. post process. std::map> algor_to_boxes; { for (auto algor_iter = algors.begin(); algor_iter != algors.end(); ++algor_iter) algor_to_boxes[*algor_iter] = std::vector(); } for (unsigned i = 0; i < model_result.obj_count_; ++i) { auto &box = model_result.obj_results_[i]; const label_t label = static_cast(box.obj_index); // LOG_TRACE("task id is {} obj_id {} label {} score {}", task_id, obj_id, label, box.obj_score); if (!is_valid_label(label)) continue; for (auto algor_type_iter = algors.begin(); algor_type_iter != algors.end(); ++algor_type_iter) { const algorithm_type_t algor_type = *algor_type_iter; // LOG_TRACE("task id is {} algor type is {} obj_id {}", task_id, int(algor_type), obj_id); auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type); if (!algor_param_wrap) { LOG_ERROR("{} is nullptr when get algor param from task_param", task_id); continue; } // LOG_TRACE("task id is {} algor type is {} obj_id {} label {} score {}", task_id, int(algor_type), obj_id, label, box.obj_score); if (!is_valid_box(box.obj_score, static_cast(box.obj_index), algor_type, algor_param_wrap)) continue; LOG_TRACE("task id is {} algor type is {} obj_id {}", task_id, int(algor_type), obj_id); box_t c_box; { c_box.cls = box.obj_index; c_box.score = box.obj_score; /* c_box.top = box.obj_rect.top_ + det_result.box.top; c_box.left = box.obj_rect.left_ + det_result.box.left; // c_box.right = c_box.left + box.obj_rect.left_ + box.obj_rect.width_; // c_box.bottom = c_box.top + box.obj_rect.top_ + box.obj_rect.height_; c_box.right = c_box.left + box.obj_rect.width_; // modified byzsh 220803 c_box.bottom = c_box.top + box.obj_rect.height_; // modified byzsh 220803*/ c_box.top = det_result.box.top; c_box.left = det_result.box.left; c_box.right = det_result.box.right; c_box.bottom = det_result.box.bottom; // modified byzsh 220803 此处调整为了行人框，后续可根据需要换回小框 } LOG_TRACE(" box {} ltrb [{} {} {} {}] score {} label {} obj_id {}", det_result.box.id, c_box.left, c_box.top, c_box.right, c_box.bottom, c_box.score, c_box.cls, obj_id); algor_to_boxes[algor_type].emplace_back(std::move(c_box)); // only save max confidence box? } } //! b. alarm strategy. { for (auto iter = algor_to_boxes.begin(); iter != algor_to_boxes.end(); ++iter) { std::vector &boxes = iter->second; const algorithm_type_t algor_type = iter->first; //! filter. if (not_semantic_algor_type_list_.count(algor_type)) { if (boxes.size()) continue; } else { if (boxes.empty()) continue; } // 221009 byzsh记录10条即可------------------------------------------------------------------------------ unique_obj_id_t unique_id_{obj_id, task_id}; if (id_to_results_.find(unique_id_) != id_to_results_.end() && id_to_results_[unique_id_].size() >= 10) continue; //------------------------------------------------------------------------------------------------------ obj_key_t obj_key{obj_id, task_id, algor_type}; LOG_TRACE("================> task_id {} algor_type {} obj_id {}", task_id, int(algor_type), obj_id); auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type); //! if (!update_and_get_mn_status(obj_key, label_t::PLACEHOLDER, /*do_erase=*/false, obj_key_to_mn_strategy_, algor_param_wrap)) continue; LOG_TRACE("22222222222 task_id {} algor_type {} obj_id {}", task_id, int(algor_type), obj_id); result_data_t result_data; { result_data.obj_id = obj_id; result_data.task_id = task_id; } if (not_semantic_algor_type_list_.count(algor_type)) { //! TODO: 使用行人box + 安全帽score?? 如果没有安全帽呢 result_data.box = det_result.box; result_data.box.score = 0.0f; } else { //! save maximum confidence box. auto max_ele = std::max_element(std::begin(boxes), std::end(boxes), [](box_t const &lhs, box_t const &rhs) { return lhs.score < rhs.score; }); result_data.box = *max_ele; // use move segmantic? } #ifdef _USE_SHALLOW_COPY { result.ori_img = src_img; result_data.roi_img = flattened_imgs[n]; } #else { { sy_img img; { img.c_ = src_img.c_; img.h_ = src_img.h_; img.w_ = src_img.w_; img.data_ = nullptr; } const unsigned size = img.c_ * img.h_ * img.w_; img.data_ = new unsigned char[size]; // img.data_ = new std::remove_pointer[size]; CHECK(cudaMemcpy(img.data_, src_img.data_, size * sizeof(unsigned char), cudaMemcpyDeviceToHost)); result_data.ori_img = std::move(img); result_data.ori_img_is_in_gpu = false; } { auto &src_img = flattened_imgs[n]; sy_img img; { img.c_ = src_img.c_; img.h_ = src_img.h_; img.w_ = src_img.w_; img.data_ = nullptr; } const unsigned size = img.c_ * img.h_ * img.w_; img.data_ = new unsigned char[size]; CHECK(cudaMemcpy(img.data_, src_img.data_, size * sizeof(unsigned char), cudaMemcpyDeviceToHost)); result_data.roi_img = std::move(img); // deep copy. // result_data.ori_img_is_in_gpu = false; result_data.roi_img_is_in_gpu = false; //221009 } } #endif is_alarm = true; result_data.algorithm_type_seq.emplace(algor_type); //------------------------------------------------------- // if (result_data.roi_img.data_ != nullptr) { // if (result_data.roi_img_is_in_gpu) { // CHECK(cudaFree((void *)(result_data.roi_img.data_)));} // else { //221007 // LOG_TRACE("PEDESTRIAN free roi cpu memory."); // delete[] result_data.roi_img.data_; // } // result_data.roi_img.data_ = nullptr; // } // if (result_data.ori_img.data_ != nullptr) { // if (result_data.ori_img_is_in_gpu) { // CHECK(cudaFree((void *)(result_data.ori_img.data_)));} // else { //221007 // LOG_TRACE("PEDESTRIAN free ori cpu memory."); // delete[] result_data.ori_img.data_; // } // result_data.ori_img.data_ = nullptr; // } //---------------------------------------------------------- id_to_results_[unique_obj_id_t{obj_id, task_id}].emplace_back(std::move(result_data)); // LOG_TRACE("has alarm task_id {} obj_id {} size {}", task_id, obj_id, id_to_results_[unique_id_].size()); } } } } //! save desc info. for (auto &img: flattened_imgs) { // LOG_TRACE("free gpu memory xxx"); // PRINT_CHECK(cudaFree(img.data_)); if (cudaSuccess != cudaFree(img.data_)) { LOG_ERROR("free image failred."); } void *ptr = nullptr; if (cudaSuccess != cudaFree(ptr)) { LOG_ERROR("free nullptr failed."); } } return true; } void PedestrianSafetyDetector::force_release_result(const task_id_t &task_id) { for (auto iter = id_to_results_.begin(); iter != id_to_results_.end();) { const auto &key = iter->first; auto &values = iter->second; if (key.task_id == task_id) { for (auto &value: values) { if (value.roi_img.data_) { LOG_TRACE("will free roi snap in gpu {}.", value.roi_img_is_in_gpu); if (value.roi_img_is_in_gpu) { CHECK(cudaFree(value.roi_img.data_)); } else { delete[]value.roi_img.data_; } value.roi_img.data_ = nullptr; } if (value.ori_img.data_) { LOG_TRACE("will free ori snap in gpu {}.", value.ori_img_is_in_gpu); if (value.ori_img_is_in_gpu) { CHECK(cudaFree(value.ori_img.data_)); } else { delete[]value.ori_img.data_; } value.ori_img.data_ = nullptr; } } values.clear(); iter = id_to_results_.erase(iter); } else { ++iter; } } } } // namespace pedestrian_safety_det } // namespace ai_engine_module