Hu Chunming / src

/*
 * File: pedestrian_safety_det.cpp
 * Created Date: Tuesday February 15th 2022
 * Author: yangzilong (yangzilong@objecteye.com)
 * Description:
 * -----
 * Last Modified: Tuesday, 15th February 2022 6:19:32 pm
 * Modified By: yangzilong (yangzilong@objecteye.com>)
 * -----
 * Copyright 2022
 */


#include "./pedestrian_safety_det.hpp"
#include "../reprocessing_module/CropImg.h"


namespace ai_engine_module {
    namespace pedestrian_safety_det {


#define INPUT_WIDTH 320
#define INPUT_HEIGHT 320
#define INPUT_CHANNEL 3

        static std::set<algorithm_type_t> algor_type_list_ = {
                algorithm_type_t::SMOKING_DET,
                algorithm_type_t::CALL_PHONE_DET,
                algorithm_type_t::NO_REFLECTIVE_CLOTHING,
                algorithm_type_t::NO_SAFETY_HELMET,
        };


        static std::set<algorithm_type_t> not_semantic_algor_type_list_ =
                {
                        algorithm_type_t::NO_REFLECTIVE_CLOTHING,
                        algorithm_type_t::NO_SAFETY_HELMET,
                };


        inline bool is_valid_label(const label_t &label) {
            return ((label == label_t::helmet) ||
                    (label == label_t::phone) ||
                    (label == label_t::smoke) ||
                    (label == label_t::vest));
        }

        inline bool is_support_algorithm_type(const algorithm_type_t &algor_type) {
            return algor_type_list_.count(algor_type) > 0;
        }


        // ############################################################ //
        // !                Auxiliary Function                        ! //
        // ############################################################ //

        std::set<algorithm_type_t> task_id_to_algorithm_type_seq(const task_id_t &task_id,
                                                                 task_param_manager *const task_param) {
            std::set<algorithm_type_t> seq;
            auto &&algor_map = task_param->get_task_other_param(task_id);
            if (algor_map) {
                // LOG_TRACE("task id is {} size algor type {}", task_id, algor_map->size());
                for (auto iter = algor_map->begin(); iter != algor_map->end(); ++iter) {
                    if (algor_type_list_.count(iter->first) > 0)
                        seq.emplace(iter->first);
                }
            }
            return seq;  // N(RVO)
        }


        pedestrian_safety_det::label_t algor_type_to_label(const algorithm_type_t &algor_type) {
            using namespace pedestrian_safety_det;
            switch (algor_type) {
                case algorithm_type_t::NO_SAFETY_HELMET:
                    return label_t::helmet;
                case algorithm_type_t::CALL_PHONE_DET:
                    return label_t::phone;
                case algorithm_type_t::SMOKING_DET:
                    return label_t::smoke;
                case algorithm_type_t::NO_REFLECTIVE_CLOTHING:
                    return label_t::vest;  //
                default:
                    return label_t::PLACEHOLDER;
            }
        }

        algorithm_type_t label_to_algor_type(const label_t &label) {
            using namespace pedestrian_safety_det;
            switch (label) {
                case label_t::helmet:
                    return algorithm_type_t::NO_SAFETY_HELMET;
                case label_t::phone:
                    return algorithm_type_t::CALL_PHONE_DET;
                case label_t::smoke:
                    return algorithm_type_t::SMOKING_DET;
                case label_t::vest:
                    return algorithm_type_t::NO_REFLECTIVE_CLOTHING;
                default:
                    return algorithm_type_t::PLACEHOLDER;
            }
        }

        size_t PedestrianSafetyDetector::get_number_of_results() {
            size_t res{0};
            for (auto iter = id_to_results_.cbegin(); iter != id_to_results_.cend(); ++iter) {
                res += iter->second.size();
            }
            return res;
        }

        bool is_valid_box(const int top, const int left, const int right, const int bottom, const float score,
                          const algorithm_type_t &algor_type,
                          const task_param_manager::algo_param_type_t_ *params_ptr = nullptr) {
            if (!params_ptr)
                return false;

            if (!snapshot_legal_inarea(params_ptr->basic_param->algor_valid_rect, left, top, right, bottom))
                return false;

            if (params_ptr->algor_param == nullptr)
                return false;


            const unsigned width = right - left;
            const unsigned height = bottom - top;

            if (width == 0 || height == 0)
                return false;

            //! TODO: use switch to replace.
            using data_t = algor_config_param_pedestrian_safety_detector_basic;
            data_t *algor_params_ptr = (data_t *) (params_ptr->algor_param);

            if (score < algor_params_ptr->pedestrian_confidence_threshold ||
                width < algor_params_ptr->pedestrian_min_width ||
                height < algor_params_ptr->pedestrian_min_height)
                return false;


            return true;
        }


        bool is_valid_box(const float score, const label_t &label,
                          const algorithm_type_t &algor_type,
                          const task_param_manager::algo_param_type_t_ *params_ptr = nullptr) {
            if (!params_ptr)
                return false;

            if (params_ptr->algor_param == nullptr)
                return false;

            //! TODO: use switch to replace.
            using data_t = algor_config_param_pedestrian_safety_detector_basic;
            data_t *algor_params_ptr = (data_t *) (params_ptr->algor_param);
            if (label != algor_type_to_label(algor_type) || score < algor_params_ptr->conf_threshold)
                return false;
            return true;
        }

        bool update_and_get_mn_status(const obj_key_t &obj_key, const label_t &label, const bool do_erase,
                                      obj_key_to_mn_strategy_t &obj_key_to_mn_strategy,
                                      const task_param_manager::algo_param_type_t_ *params_ptr = nullptr) {
            if (params_ptr == nullptr) {
                LOG_ERROR(" params_ptr is null.");
                return false;
            }

            if (params_ptr->algor_param == nullptr) {
                LOG_ERROR(" params -> algor_param is null.");
                return false;
            }

            //! TODO: use switch to replace.
            using data_t = algor_config_param_pedestrian_safety_detector_basic;
            data_t *algor_params_ptr = (data_t *) (params_ptr->algor_param);
            using label_t = pedestrian_safety_det::label_t;

            auto &mn_strategy = obj_key_to_mn_strategy[obj_key];

            mn_strategy.set_target_m(algor_params_ptr->m);
            mn_strategy.set_target_n(algor_params_ptr->n);

            if (label != label_t::PLACEHOLDER) {
                label_t target_label = algor_type_to_label(obj_key.algor_type);
                if (label_t::PLACEHOLDER == target_label)
                    return false;
                return mn_strategy.update_and_check(label, target_label, /*do_reset=*/do_erase);
            }

            bool ret = mn_strategy.update_and_check(/*do_erase=*/do_erase);
            return ret;
        }


        // ############################################################ //
        // !                    Class Member                          ! //
        // ############################################################ //


        PedestrianSafetyDetector::PedestrianSafetyDetector()
                : task_param_manager_(nullptr), handle_(nullptr), batch_size_(0), initied_(false) {

        }

        PedestrianSafetyDetector::~PedestrianSafetyDetector() {
            if (handle_ != nullptr) {
                hat_callsmoke_vestuniform_det_release(&handle_);
                if (!handle_) {
                    delete handle_;
                    handle_ = nullptr;
                }
            }
        }

        bool PedestrianSafetyDetector::init(const hat_callsmoke_vestuniform_det_param &initied_params) {
            int status = 0;
            if (!(initied_ = (0 == (status = hat_callsmoke_vestuniform_det_init(&handle_, initied_params))))) {
                LOG_ERROR("Init PedestrianSafetyDetectorSdk failed error code is {}", status);
                return false;
            }

            batch_size_ = initied_params.max_batch;

            if (!task_param_manager_)
                task_param_manager_ = task_param_manager::getInstance();
            return initied_;
        }

        bool PedestrianSafetyDetector::is_support(const algorithm_type_t &algor_type) const noexcept {
            return algor_type_list_.count(algor_type) > 0;
        }

        bool PedestrianSafetyDetector::check_initied() const {
            if (!initied_)
                LOG_ERROR("[%s:%d] call init function please.", __FILE__, __LINE__);
            return initied_;
        }

        std::shared_ptr<results_data_t>
        PedestrianSafetyDetector::get_results_by_id(const unique_obj_id_t &id, bool do_erase) {
            auto it = id_to_results_.find(id);
            if (it == id_to_results_.end())
                return nullptr;
            // std::shared_ptr<result_data_t> res(&(it->second));
            std::shared_ptr<results_data_t> res = std::make_shared<results_data_t>(std::move(it->second));
            if (do_erase)
                id_to_results_.erase(id);
            return res;
        }

        bool PedestrianSafetyDetector::update_mstreams(const std::set<task_id_t> &tasks_id,
                                                       const sy_img *det_input_images,
                                                       const std::vector<onelevel_det_result> &det_results) {

            if (!check_initied())
                return false;

            if (tasks_id.empty()) {
                LOG_DEBUG("task_id is empty.");
                return false;
            }

            struct stream_idx_and_algor_seq_t {
                unsigned stream_idx;
                std::set<algorithm_type_t> algors;
            };

            // LOG_TRACE("number of tasks_id is {}", tasks_id.size());

            unsigned flattened_idx = 0;
            //! 记录每个box对应的算法以及流id.
            std::map<unsigned, stream_idx_and_algor_seq_t> flattened_idx_to_algor_seq;


            /* 1. Crop & keep some interest class. */
            unsigned stream_idx = 0;
            std::vector<sy_img> flattened_imgs(0);
            std::vector<input_data_wrap_t> flattened_interest_data(0);  //
            for (auto iter = tasks_id.begin(); iter != tasks_id.end(); ++iter, ++stream_idx) {
                task_id_t task_id = *iter;
                auto &src_img = det_input_images[stream_idx];
                auto &boxes_of_one_image = det_results[stream_idx].obj;
                for (int i = 0; i < det_results[stream_idx].obj_count; ++i)  //! loop per box.
                {
                    auto &box = boxes_of_one_image[i];
                    if (static_cast<det_class_label_t>(box.index) != det_class_label_t::HUMAN)
                        continue;

                    //! get cropped box.
                    input_data_wrap_t data;
                    int top = std::max(int(box.top - (IMAGE_CROP_EXPAND_RATIO * box.top)), 0);
                    int left = std::max(int(box.left - (IMAGE_CROP_EXPAND_RATIO * box.left)), 0);
                    int right = std::min(int(box.right + (IMAGE_CROP_EXPAND_RATIO * box.right)), src_img.w_);
                    int bottom = std::min(int(box.bottom + (IMAGE_CROP_EXPAND_RATIO * box.bottom)), src_img.h_);

                    //! loop per algor from set.
                    stream_idx_and_algor_seq_t stream_idx_and_algor_seq{stream_idx, {}};

                    std::set<algorithm_type_t> algorithm_type_seq = task_id_to_algorithm_type_seq(task_id,
                                                                                                  task_param_manager_);  // N(RVO).
                    for (auto algor_iter = algorithm_type_seq.begin();
                         algor_iter != algorithm_type_seq.end(); ++algor_iter) {
                        const algorithm_type_t algor_type = *algor_iter;
                        auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type);
                        if (!algor_param_wrap) {
                            LOG_ERROR("{} is nullptr when get algor param from task_param", task_id.c_str());
                            continue;
                        }

                        if (!is_valid_box(top, left, right, bottom, box.confidence, algor_type, algor_param_wrap))
                            continue;

                        stream_idx_and_algor_seq.algors.emplace(algor_type);
                    }

                    if (stream_idx_and_algor_seq.algors.empty())
                        continue;


                    int width = right - left;
                    int height = bottom - top;

                    data.box.top = top;
                    data.box.left = left;
                    data.box.right = right;
                    data.box.bottom = bottom;
                    data.taskId = task_id;
                    data.objId = box.id;

                    sy_img img;
                    {
                        img.w_ = width;
                        img.h_ = height;
                        img.c_ = src_img.c_;
                        img.data_ = nullptr;
                    }

                    cudaError_t cuda_status;
                    const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char);
                    if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void **) &img.data_, nbytes))) {
                        LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ",
                                  cudaGetErrorString(cuda_status),
                                  nbytes, nbytes / (1024 * 1024));
                        continue;
                    }

                    //! TODO: use long short edge resize to replace.
                    if (CUDA_SUCCESS !=
                        (cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, left,
                                                              top,
                                                              width, height))) {
                        LOG_ERROR("Crop image GPU failed error is {} wh is [{}, {}] ltrb is [{} {} {} {}]",
                                  cudaGetErrorString(cuda_status), src_img.w_, src_img.h_,
                                  data.box.left, data.box.top, data.box.right, data.box.bottom);
                        CHECK(cudaFree(img.data_));
                        continue;
                    }

                    flattened_imgs.emplace_back(std::move(img));
                    flattened_interest_data.emplace_back(std::move(data));
                    flattened_idx_to_algor_seq[flattened_idx++] = std::move(stream_idx_and_algor_seq);
                }
            }

            if (flattened_imgs.empty())
                return true;

            // printf("[Ckpt1] number of f_imgs f_data is %lu %lu\n", flattened_imgs.size(), flattened_interest_data.size());
            LOG_TRACE("[Ckpt1] number of f_imgs f_data is {} {}", flattened_imgs.size(),
                      flattened_interest_data.size());

            /* 2. do inference and get result. */
            const unsigned n_input_image = flattened_imgs.size();
            LOG_TRACE("input image size {}", n_input_image);
            hat_callsmoke_vestuniform_det_result model_results[n_input_image];
            {
                const unsigned stride = batch_size_;
                const unsigned steps = (n_input_image + stride - 1) / stride;


                for (unsigned step = 0; step < steps; ++step) {
                    const unsigned offset = step * stride;
                    const unsigned batch_size = (step == steps - 1) ? n_input_image - offset : stride;

                    // printf("  Step: [%d] start_idx: %u batch_size: %u\n", step, offset, batch_size);
                    LOG_TRACE("  Step: [{}] start_idx: {} batch_size: {}", step, offset, batch_size);
                    hat_callsmoke_vestuniform_det_batch(handle_, flattened_imgs.data() + offset, batch_size,
                                                        model_results + offset);
                }
            }

            // printf("[Ckpt2] number of f_imgs f_data is %lu %lu\n", flattened_imgs.size(), flattened_interest_data.size());
            LOG_TRACE("[Ckpt2] number of f_imgs f_data is {} {}", flattened_imgs.size(),
                      flattened_interest_data.size());

            // return true;

            /* 3. postprocess. */
            {
                //! loop per box output.
                for (unsigned n = 0; n < n_input_image; ++n) {
                    auto &det_result = flattened_interest_data[n];
                    auto &obj_id = det_result.objId;
                    auto &task_id = det_result.taskId;

                    auto &stream_idx_and_algor_seq = flattened_idx_to_algor_seq[n];
                    auto &steram_idx = stream_idx_and_algor_seq.stream_idx;
                    auto &src_img = det_input_images[steram_idx];
                    auto &algors = stream_idx_and_algor_seq.algors;

                    auto &model_result = model_results[n];

                    //! loop per cropped image result.
                    bool is_alarm = false;
                    // LOG_DEBUG("[{}] box count {}", task_id, model_result.obj_count_);

                    //! a. post process.
                    std::map<algorithm_type_t, std::vector<box_t>> algor_to_boxes;
                    {
                        for (auto algor_iter = algors.begin(); algor_iter != algors.end(); ++algor_iter)
                            algor_to_boxes[*algor_iter] = std::vector<box_t>();
                    }
                    for (unsigned i = 0; i < model_result.obj_count_; ++i) {
                        auto &box = model_result.obj_results_[i];
                        const label_t label = static_cast<label_t>(box.obj_index);

                        // LOG_TRACE("task id is {} obj_id {} label {} score {}", task_id, obj_id, label, box.obj_score);
                        if (!is_valid_label(label))
                            continue;

                        for (auto algor_type_iter = algors.begin();
                             algor_type_iter != algors.end(); ++algor_type_iter) {
                            const algorithm_type_t algor_type = *algor_type_iter;
                            // LOG_TRACE("task id is {} algor type is {} obj_id {}", task_id, int(algor_type), obj_id);
                            auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type);
                            if (!algor_param_wrap) {
                                LOG_ERROR("{} is nullptr when get algor param from task_param", task_id);
                                continue;
                            }
                            // LOG_TRACE("task id is {} algor type is {} obj_id {} label {} score {}", task_id, int(algor_type), obj_id, label, box.obj_score);
                            if (!is_valid_box(box.obj_score, static_cast<label_t>(box.obj_index), algor_type,
                                              algor_param_wrap))
                                continue;
                            LOG_TRACE("task id is {} algor type is {} obj_id {}", task_id, int(algor_type), obj_id);
                            box_t c_box;
                            {
                                c_box.cls = box.obj_index;
                                c_box.score = box.obj_score;
                                /*
                                c_box.top = box.obj_rect.top_ + det_result.box.top;
                                c_box.left = box.obj_rect.left_ + det_result.box.left;
                                // c_box.right = c_box.left + box.obj_rect.left_ + box.obj_rect.width_;
                                // c_box.bottom = c_box.top + box.obj_rect.top_ + box.obj_rect.height_;
                                c_box.right = c_box.left + box.obj_rect.width_; // modified byzsh 220803 
                                c_box.bottom = c_box.top + box.obj_rect.height_; // modified byzsh 220803*/
                                
                                c_box.top = det_result.box.top;
                                c_box.left = det_result.box.left;
                                c_box.right = det_result.box.right; 
                                c_box.bottom = det_result.box.bottom; // modified byzsh 220803 此处调整为了行人框，后续可根据需要换回小框
                            }
                            LOG_TRACE("  box {} ltrb [{} {} {} {}] score {} label {} obj_id {}", det_result.box.id, c_box.left,
                                      c_box.top, c_box.right, c_box.bottom, c_box.score, c_box.cls, obj_id);
                            algor_to_boxes[algor_type].emplace_back(std::move(c_box));  // only save max confidence box?
                        }
                    }



                    //! b. alarm strategy.
                    {
                        for (auto iter = algor_to_boxes.begin(); iter != algor_to_boxes.end(); ++iter) {
                            std::vector<box_t> &boxes = iter->second;
                            const algorithm_type_t algor_type = iter->first;

                            //! filter.
                            if (not_semantic_algor_type_list_.count(algor_type)) {
                                if (boxes.size())
                                    continue;
                            } else {
                                if (boxes.empty())
                                    continue;
                            }

                            
                            // 221009 byzsh记录10条即可------------------------------------------------------------------------------
                            unique_obj_id_t unique_id_{obj_id, task_id};
                            if (id_to_results_.find(unique_id_) != id_to_results_.end() && id_to_results_[unique_id_].size() >= 10)
                                continue;
                            //------------------------------------------------------------------------------------------------------
                            
                            
                            obj_key_t obj_key{obj_id, task_id, algor_type};


                            LOG_TRACE("================> task_id {} algor_type {} obj_id {}", task_id, int(algor_type), obj_id);
                            auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type);

                            //!
                            if (!update_and_get_mn_status(obj_key, label_t::PLACEHOLDER, /*do_erase=*/false,
                                                          obj_key_to_mn_strategy_, algor_param_wrap))
                                continue;
                            LOG_TRACE("22222222222 task_id {} algor_type {} obj_id {}", task_id, int(algor_type), obj_id);
                            result_data_t result_data;
                            {
                                result_data.obj_id = obj_id;
                                result_data.task_id = task_id;
                            }


                            if (not_semantic_algor_type_list_.count(algor_type)) {
                                //! TODO: 使用行人box + 安全帽score?? 如果没有安全帽呢
                                result_data.box = det_result.box;
                                result_data.box.score = 0.0f;
                            } else {
                                //! save maximum confidence box.
                                auto max_ele = std::max_element(std::begin(boxes), std::end(boxes),
                                                                [](box_t const &lhs, box_t const &rhs) {
                                                                    return lhs.score < rhs.score;
                                                                });
                                result_data.box = *max_ele;  // use move segmantic?
                            }


#ifdef _USE_SHALLOW_COPY
                            {
                                result.ori_img = src_img;
                                result_data.roi_img = flattened_imgs[n];
                            }
#else
                            {
                                {
                                    sy_img img;
                                    {
                                        img.c_ = src_img.c_;
                                        img.h_ = src_img.h_;
                                        img.w_ = src_img.w_;
                                        img.data_ = nullptr;
                                    }
                                    const unsigned size = img.c_ * img.h_ * img.w_;
                                    img.data_ = new unsigned char[size];
                                    // img.data_ = new std::remove_pointer<decltype(img.data_)>[size];
                                    CHECK(cudaMemcpy(img.data_, src_img.data_, size * sizeof(unsigned char),
                                                     cudaMemcpyDeviceToHost));
                                    result_data.ori_img = std::move(img);
                                    result_data.ori_img_is_in_gpu = false;
                                }

                                {
                                    auto &src_img = flattened_imgs[n];
                                    sy_img img;
                                    {
                                        img.c_ = src_img.c_;
                                        img.h_ = src_img.h_;
                                        img.w_ = src_img.w_;
                                        img.data_ = nullptr;
                                    }
                                    const unsigned size = img.c_ * img.h_ * img.w_;
                                    img.data_ = new unsigned char[size];
                                    CHECK(cudaMemcpy(img.data_, src_img.data_, size * sizeof(unsigned char),
                                                     cudaMemcpyDeviceToHost));
                                    result_data.roi_img = std::move(img);  // deep copy.
                                    // result_data.ori_img_is_in_gpu = false;
                                    result_data.roi_img_is_in_gpu = false; //221009
                                }
                            }
#endif

                            is_alarm = true;
                            result_data.algorithm_type_seq.emplace(algor_type);

                            //-------------------------------------------------------
                            // if (result_data.roi_img.data_ != nullptr) {
                            //     if (result_data.roi_img_is_in_gpu) {
                            //         CHECK(cudaFree((void *)(result_data.roi_img.data_)));}
                            //     else { //221007
                            //         LOG_TRACE("PEDESTRIAN free roi cpu memory.");
                            //         delete[] result_data.roi_img.data_;
                            //     }
                            //     result_data.roi_img.data_ = nullptr;
                            //     }

                            //     if (result_data.ori_img.data_ != nullptr) {
                            //     if (result_data.ori_img_is_in_gpu) {
                            //         CHECK(cudaFree((void *)(result_data.ori_img.data_)));}
                            //     else { //221007
                            //         LOG_TRACE("PEDESTRIAN free ori cpu memory.");
                            //         delete[] result_data.ori_img.data_;
                            //     }
                            //     result_data.ori_img.data_ = nullptr;
                            //     }
                            //----------------------------------------------------------
                            id_to_results_[unique_obj_id_t{obj_id, task_id}].emplace_back(std::move(result_data)); //
                            LOG_TRACE("has alarm task_id {} obj_id {} size {}", task_id, obj_id, id_to_results_[unique_id_].size());
                        }
                    }

                }
            }



            //! save desc info.
            for (auto &img: flattened_imgs) {
                // LOG_TRACE("free gpu memory xxx");
                // PRINT_CHECK(cudaFree(img.data_));
                if (cudaSuccess != cudaFree(img.data_)) {
                    LOG_ERROR("free image failred.");
                }

                void *ptr = nullptr;
                if (cudaSuccess != cudaFree(ptr)) {
                    LOG_ERROR("free nullptr failed.");
                }
            }
            return true;
        }

        bool PedestrianSafetyDetector::update_mstreams2(const std::vector<task_id_t> &tasks_id,
                                                       const sy_img *det_input_images,
                                                       const std::vector<onelevel_det_result> &det_results) {

            if (!check_initied())
                return false;

            if (tasks_id.empty()) {
                LOG_DEBUG("task_id is empty.");
                return false;
            }

            struct stream_idx_and_algor_seq_t {
                unsigned stream_idx;
                std::set<algorithm_type_t> algors;
            };

            // LOG_TRACE("number of tasks_id is {}", tasks_id.size());

            unsigned flattened_idx = 0;
            //! 记录每个box对应的算法以及流id.
            std::map<unsigned, stream_idx_and_algor_seq_t> flattened_idx_to_algor_seq;


            /* 1. Crop & keep some interest class. */
            unsigned stream_idx = 0;
            std::vector<sy_img> flattened_imgs(0);
            std::vector<input_data_wrap_t> flattened_interest_data(0);  //
            for (auto iter = tasks_id.begin(); iter != tasks_id.end(); ++iter, ++stream_idx) {
                task_id_t task_id = *iter;
                auto &src_img = det_input_images[stream_idx];
                auto &boxes_of_one_image = det_results[stream_idx].obj;
                for (int i = 0; i < det_results[stream_idx].obj_count; ++i)  //! loop per box.
                {
                    auto &box = boxes_of_one_image[i];
                    if (static_cast<det_class_label_t>(box.index) != det_class_label_t::HUMAN)
                        continue;

                    //! get cropped box.
                    input_data_wrap_t data;
                    int top = std::max(int(box.top - (IMAGE_CROP_EXPAND_RATIO * box.top)), 0);
                    int left = std::max(int(box.left - (IMAGE_CROP_EXPAND_RATIO * box.left)), 0);
                    int right = std::min(int(box.right + (IMAGE_CROP_EXPAND_RATIO * box.right)), src_img.w_);
                    int bottom = std::min(int(box.bottom + (IMAGE_CROP_EXPAND_RATIO * box.bottom)), src_img.h_);

                    //! loop per algor from set.
                    stream_idx_and_algor_seq_t stream_idx_and_algor_seq{stream_idx, {}};

                    std::set<algorithm_type_t> algorithm_type_seq = task_id_to_algorithm_type_seq(task_id,
                                                                                                  task_param_manager_);  // N(RVO).
                    for (auto algor_iter = algorithm_type_seq.begin();
                         algor_iter != algorithm_type_seq.end(); ++algor_iter) {
                        const algorithm_type_t algor_type = *algor_iter;
                        auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type);
                        if (!algor_param_wrap) {
                            LOG_ERROR("{} is nullptr when get algor param from task_param", task_id.c_str());
                            continue;
                        }

                        if (!is_valid_box(top, left, right, bottom, box.confidence, algor_type, algor_param_wrap))
                            continue;

                        stream_idx_and_algor_seq.algors.emplace(algor_type);
                    }

                    if (stream_idx_and_algor_seq.algors.empty())
                        continue;


                    int width = right - left;
                    int height = bottom - top;

                    data.box.top = top;
                    data.box.left = left;
                    data.box.right = right;
                    data.box.bottom = bottom;
                    data.taskId = task_id;
                    data.objId = box.id;

                    sy_img img;
                    {
                        img.w_ = width;
                        img.h_ = height;
                        img.c_ = src_img.c_;
                        img.data_ = nullptr;
                    }

                    cudaError_t cuda_status;
                    const unsigned nbytes = img.c_ * img.h_ * img.w_ * sizeof(unsigned char);
                    if (CUDA_SUCCESS != (cuda_status = cudaMalloc((void **) &img.data_, nbytes))) {
                        LOG_ERROR("cudaMalloc failed: {} malloc nbytes is {} mb is {} ",
                                  cudaGetErrorString(cuda_status),
                                  nbytes, nbytes / (1024 * 1024));
                        continue;
                    }

                    //! TODO: use long short edge resize to replace.
                    if (CUDA_SUCCESS !=
                        (cuda_status = cudacommon::CropImgGpu(src_img.data_, src_img.w_, src_img.h_, img.data_, left,
                                                              top,
                                                              width, height))) {
                        LOG_ERROR("Crop image GPU failed error is {} wh is [{}, {}] ltrb is [{} {} {} {}]",
                                  cudaGetErrorString(cuda_status), src_img.w_, src_img.h_,
                                  data.box.left, data.box.top, data.box.right, data.box.bottom);
                        CHECK(cudaFree(img.data_));
                        continue;
                    }

                    flattened_imgs.emplace_back(std::move(img));
                    flattened_interest_data.emplace_back(std::move(data));
                    flattened_idx_to_algor_seq[flattened_idx++] = std::move(stream_idx_and_algor_seq);
                }
            }

            if (flattened_imgs.empty())
                return true;

            // printf("[Ckpt1] number of f_imgs f_data is %lu %lu\n", flattened_imgs.size(), flattened_interest_data.size());
            LOG_TRACE("[Ckpt1] number of f_imgs f_data is {} {}", flattened_imgs.size(),
                      flattened_interest_data.size());

            /* 2. do inference and get result. */
            const unsigned n_input_image = flattened_imgs.size();
            LOG_TRACE("input image size {}", n_input_image);
            hat_callsmoke_vestuniform_det_result model_results[n_input_image];
            {
                const unsigned stride = batch_size_;
                const unsigned steps = (n_input_image + stride - 1) / stride;


                for (unsigned step = 0; step < steps; ++step) {
                    const unsigned offset = step * stride;
                    const unsigned batch_size = (step == steps - 1) ? n_input_image - offset : stride;

                    // printf("  Step: [%d] start_idx: %u batch_size: %u\n", step, offset, batch_size);
                    LOG_TRACE("  Step: [{}] start_idx: {} batch_size: {}", step, offset, batch_size);
                    hat_callsmoke_vestuniform_det_batch(handle_, flattened_imgs.data() + offset, batch_size,
                                                        model_results + offset);
                }
            }

            // printf("[Ckpt2] number of f_imgs f_data is %lu %lu\n", flattened_imgs.size(), flattened_interest_data.size());
            LOG_TRACE("[Ckpt2] number of f_imgs f_data is {} {}", flattened_imgs.size(),
                      flattened_interest_data.size());

            // return true;

            /* 3. postprocess. */
            {
                //! loop per box output.
                for (unsigned n = 0; n < n_input_image; ++n) {
                    auto &det_result = flattened_interest_data[n];
                    auto &obj_id = det_result.objId;
                    auto &task_id = det_result.taskId;

                    auto &stream_idx_and_algor_seq = flattened_idx_to_algor_seq[n];
                    auto &steram_idx = stream_idx_and_algor_seq.stream_idx;
                    auto &src_img = det_input_images[steram_idx];
                    auto &algors = stream_idx_and_algor_seq.algors;

                    auto &model_result = model_results[n];

                    //! loop per cropped image result.
                    bool is_alarm = false;
                    // LOG_DEBUG("[{}] box count {}", task_id, model_result.obj_count_);

                    //! a. post process.
                    std::map<algorithm_type_t, std::vector<box_t>> algor_to_boxes;
                    {
                        for (auto algor_iter = algors.begin(); algor_iter != algors.end(); ++algor_iter)
                            algor_to_boxes[*algor_iter] = std::vector<box_t>();
                    }
                    for (unsigned i = 0; i < model_result.obj_count_; ++i) {
                        auto &box = model_result.obj_results_[i];
                        const label_t label = static_cast<label_t>(box.obj_index);

                        // LOG_TRACE("task id is {} obj_id {} label {} score {}", task_id, obj_id, label, box.obj_score);
                        if (!is_valid_label(label))
                            continue;

                        for (auto algor_type_iter = algors.begin();
                             algor_type_iter != algors.end(); ++algor_type_iter) {
                            const algorithm_type_t algor_type = *algor_type_iter;
                            // LOG_TRACE("task id is {} algor type is {} obj_id {}", task_id, int(algor_type), obj_id);
                            auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type);
                            if (!algor_param_wrap) {
                                LOG_ERROR("{} is nullptr when get algor param from task_param", task_id);
                                continue;
                            }
                            // LOG_TRACE("task id is {} algor type is {} obj_id {} label {} score {}", task_id, int(algor_type), obj_id, label, box.obj_score);
                            if (!is_valid_box(box.obj_score, static_cast<label_t>(box.obj_index), algor_type,
                                              algor_param_wrap))
                                continue;
                            LOG_TRACE("task id is {} algor type is {} obj_id {}", task_id, int(algor_type), obj_id);
                            box_t c_box;
                            {
                                c_box.cls = box.obj_index;
                                c_box.score = box.obj_score;
                                /*
                                c_box.top = box.obj_rect.top_ + det_result.box.top;
                                c_box.left = box.obj_rect.left_ + det_result.box.left;
                                // c_box.right = c_box.left + box.obj_rect.left_ + box.obj_rect.width_;
                                // c_box.bottom = c_box.top + box.obj_rect.top_ + box.obj_rect.height_;
                                c_box.right = c_box.left + box.obj_rect.width_; // modified byzsh 220803 
                                c_box.bottom = c_box.top + box.obj_rect.height_; // modified byzsh 220803*/
                                
                                c_box.top = det_result.box.top;
                                c_box.left = det_result.box.left;
                                c_box.right = det_result.box.right; 
                                c_box.bottom = det_result.box.bottom; // modified byzsh 220803 此处调整为了行人框，后续可根据需要换回小框
                            }
                            LOG_TRACE("  box {} ltrb [{} {} {} {}] score {} label {} obj_id {}", det_result.box.id, c_box.left,
                                      c_box.top, c_box.right, c_box.bottom, c_box.score, c_box.cls, obj_id);
                            algor_to_boxes[algor_type].emplace_back(std::move(c_box));  // only save max confidence box?
                        }
                    }



                    //! b. alarm strategy.
                    {
                        for (auto iter = algor_to_boxes.begin(); iter != algor_to_boxes.end(); ++iter) {
                            std::vector<box_t> &boxes = iter->second;
                            const algorithm_type_t algor_type = iter->first;

                            //! filter.
                            if (not_semantic_algor_type_list_.count(algor_type)) {
                                if (boxes.size())
                                    continue;
                            } else {
                                if (boxes.empty())
                                    continue;
                            }

                            
                            // 221009 byzsh记录10条即可------------------------------------------------------------------------------
                            unique_obj_id_t unique_id_{obj_id, task_id};
                            if (id_to_results_.find(unique_id_) != id_to_results_.end() && id_to_results_[unique_id_].size() >= 10)
                                continue;
                            //------------------------------------------------------------------------------------------------------
                            
                            
                            obj_key_t obj_key{obj_id, task_id, algor_type};


                            LOG_TRACE("================> task_id {} algor_type {} obj_id {}", task_id, int(algor_type), obj_id);
                            auto &&algor_param_wrap = task_param_manager_->get_task_other_param(task_id, algor_type);

                            //!
                            if (!update_and_get_mn_status(obj_key, label_t::PLACEHOLDER, /*do_erase=*/false,
                                                          obj_key_to_mn_strategy_, algor_param_wrap))
                                continue;
                            LOG_TRACE("22222222222 task_id {} algor_type {} obj_id {}", task_id, int(algor_type), obj_id);
                            result_data_t result_data;
                            {
                                result_data.obj_id = obj_id;
                                result_data.task_id = task_id;
                            }


                            if (not_semantic_algor_type_list_.count(algor_type)) {
                                //! TODO: 使用行人box + 安全帽score?? 如果没有安全帽呢
                                result_data.box = det_result.box;
                                result_data.box.score = 0.0f;
                            } else {
                                //! save maximum confidence box.
                                auto max_ele = std::max_element(std::begin(boxes), std::end(boxes),
                                                                [](box_t const &lhs, box_t const &rhs) {
                                                                    return lhs.score < rhs.score;
                                                                });
                                result_data.box = *max_ele;  // use move segmantic?
                            }


#ifdef _USE_SHALLOW_COPY
                            {
                                result.ori_img = src_img;
                                result_data.roi_img = flattened_imgs[n];
                            }
#else
                            {
                                {
                                    sy_img img;
                                    {
                                        img.c_ = src_img.c_;
                                        img.h_ = src_img.h_;
                                        img.w_ = src_img.w_;
                                        img.data_ = nullptr;
                                    }
                                    const unsigned size = img.c_ * img.h_ * img.w_;
                                    img.data_ = new unsigned char[size];
                                    // img.data_ = new std::remove_pointer<decltype(img.data_)>[size];
                                    CHECK(cudaMemcpy(img.data_, src_img.data_, size * sizeof(unsigned char),
                                                     cudaMemcpyDeviceToHost));
                                    result_data.ori_img = std::move(img);
                                    result_data.ori_img_is_in_gpu = false;
                                }

                                {
                                    auto &src_img = flattened_imgs[n];
                                    sy_img img;
                                    {
                                        img.c_ = src_img.c_;
                                        img.h_ = src_img.h_;
                                        img.w_ = src_img.w_;
                                        img.data_ = nullptr;
                                    }
                                    const unsigned size = img.c_ * img.h_ * img.w_;
                                    img.data_ = new unsigned char[size];
                                    CHECK(cudaMemcpy(img.data_, src_img.data_, size * sizeof(unsigned char),
                                                     cudaMemcpyDeviceToHost));
                                    result_data.roi_img = std::move(img);  // deep copy.
                                    // result_data.ori_img_is_in_gpu = false;
                                    result_data.roi_img_is_in_gpu = false; //221009
                                }
                            }
#endif

                            is_alarm = true;
                            result_data.algorithm_type_seq.emplace(algor_type);

                            //-------------------------------------------------------
                            // if (result_data.roi_img.data_ != nullptr) {
                            //     if (result_data.roi_img_is_in_gpu) {
                            //         CHECK(cudaFree((void *)(result_data.roi_img.data_)));}
                            //     else { //221007
                            //         LOG_TRACE("PEDESTRIAN free roi cpu memory.");
                            //         delete[] result_data.roi_img.data_;
                            //     }
                            //     result_data.roi_img.data_ = nullptr;
                            //     }

                            //     if (result_data.ori_img.data_ != nullptr) {
                            //     if (result_data.ori_img_is_in_gpu) {
                            //         CHECK(cudaFree((void *)(result_data.ori_img.data_)));}
                            //     else { //221007
                            //         LOG_TRACE("PEDESTRIAN free ori cpu memory.");
                            //         delete[] result_data.ori_img.data_;
                            //     }
                            //     result_data.ori_img.data_ = nullptr;
                            //     }
                            //----------------------------------------------------------
                            id_to_results_[unique_obj_id_t{obj_id, task_id}].emplace_back(std::move(result_data)); //
                            LOG_TRACE("has alarm task_id {} obj_id {} size {}", task_id, obj_id, id_to_results_[unique_id_].size());
                        }
                    }

                }
            }



            //! save desc info.
            for (auto &img: flattened_imgs) {
                // LOG_TRACE("free gpu memory xxx");
                // PRINT_CHECK(cudaFree(img.data_));
                if (cudaSuccess != cudaFree(img.data_)) {
                    LOG_ERROR("free image failred.");
                }

                void *ptr = nullptr;
                if (cudaSuccess != cudaFree(ptr)) {
                    LOG_ERROR("free nullptr failed.");
                }
            }
            return true;
        }

        void PedestrianSafetyDetector::force_release_result(const task_id_t &task_id) {
            for (auto iter = id_to_results_.begin(); iter != id_to_results_.end();) {
                const auto &key = iter->first;
                auto &values = iter->second;

                if (key.task_id == task_id) {
                    for (auto &value: values) {
                        if (value.roi_img.data_) {
                            LOG_TRACE("will free roi snap in gpu {}.", value.roi_img_is_in_gpu);
                            if (value.roi_img_is_in_gpu) {
                                CHECK(cudaFree(value.roi_img.data_));
                            } else {
                                delete[]value.roi_img.data_;
                            }
                            value.roi_img.data_ = nullptr;
                        }

                        if (value.ori_img.data_) {
                            LOG_TRACE("will free ori snap in gpu {}.", value.ori_img_is_in_gpu);
                            if (value.ori_img_is_in_gpu) {
                                CHECK(cudaFree(value.ori_img.data_));
                            } else {
                                delete[]value.ori_img.data_;
                            }
                            value.ori_img.data_ = nullptr;
                        }
                    }
                    values.clear();
                    iter = id_to_results_.erase(iter);
                } else {
                    ++iter;
                }
            }

        }

    }  // namespace pedestrian_safety_det
} // namespace ai_engine_module