diff --git a/bin/libvpt_det.so b/bin/libvpt_det.so index 36c7638..571402f 100644 --- a/bin/libvpt_det.so +++ b/bin/libvpt_det.so diff --git a/src/ai_engine_module/ai_engine_header.h b/src/ai_engine_module/ai_engine_header.h new file mode 100644 index 0000000..a21779c --- /dev/null +++ b/src/ai_engine_module/ai_engine_header.h @@ -0,0 +1,86 @@ +/* + * @Author: yangzilong + * @Date: 2021-12-01 15:14:02 + * @Last Modified by: yangzilong + * @Last Modified time: Do not edit + * @Email: yangzilong@objecteye.com + * @Description: + */ + +#pragma once + +#include +#include +#include +#include "../ai_platform/det_obj_header.h" +#include "../util/vpc_util.h" +#include "../ai_platform/common_header.h" + +namespace ai_engine_module { +template T clip(const T &val, const T &min, const T &max) { + if (std::is_integral::value) + return std::min(std::max(val, min), max); + return val; +} + +using obj_id_t = long; +using task_id_t = std::string; + +enum class direction_t { + NEGATIVE = 0, + POSITIVE = 1, +}; + +struct unique_obj_id_t { + obj_id_t obj_id; + task_id_t task_id; + + bool operator<(const unique_obj_id_t &obj) const { + if (obj_id < obj.obj_id) + return true; + + else if (obj_id == obj.obj_id) { + if (strcmp(task_id.c_str(), obj.task_id.c_str()) < 0) + return true; + } + return false; + } + +}; + +struct obj_key_t { + obj_id_t obj_id; + task_id_t task_id; + algorithm_type_t algor_type; + + bool operator<(const obj_key_t &obj) const { + if (obj_id < obj.obj_id) + return true; + + else if (obj_id == obj.obj_id) { + if (strcmp(task_id.c_str(), obj.task_id.c_str()) < 0) + return true; + + else if (strcmp(task_id.c_str(), obj.task_id.c_str()) == 0) { + if (static_cast(algor_type) < static_cast(obj.algor_type)) + return true; + else + return false; + } + } + return false; + } +}; + +struct trace_t { + box_t box; + point_t point; +}; + +typedef struct result_data_t { + box_t box; + acldvppPicDesc* origin_img_desc{nullptr}; + acldvppPicDesc* roi_img_desc{nullptr}; +} result_data_t; + +} // namespace ai_engine_module diff --git a/src/ai_engine_module/pedestrian_vehicle_retrograde.cpp b/src/ai_engine_module/pedestrian_vehicle_retrograde.cpp new file mode 100644 index 0000000..b777ab2 --- /dev/null +++ b/src/ai_engine_module/pedestrian_vehicle_retrograde.cpp @@ -0,0 +1,395 @@ +/* + * File: pedestrian_vehicle_retrograde.cpp + * Created Date: Tuesday February 22nd 2022 + * Author: yangzilong (yangzilong@objecteye.com) + * Description: + * ----- + * Last Modified: Tuesday, 22nd February 2022 4:38:48 pm + * Modified By: yangzilong (yangzilong@objecteye.com>) + * ----- + * Copyright 2022 + */ + +#include "./pedestrian_vehicle_retrograde.h" +#include +#include "../decoder/interface/DeviceMemory.hpp" +#include "../common/logger.hpp" +#include "../ai_platform/mvpt_process_assist.h" + +namespace ai_engine_module { +namespace pedestrian_vehicle_retrograde { + +#define MAX_TRACE_NUM 5 +#define MINIMUM_DISTANCE 10 + +static std::set algor_type_list_ = { + algorithm_type_t::PEDESTRIAN_RETROGRADE, + algorithm_type_t::VEHICLE_RETROGRADE, +}; + +std::set algor_type_to_det_label_set(const algorithm_type_t &algor_type) { + if (algorithm_type_t::PEDESTRIAN_RETROGRADE == algor_type) { + return {det_class_label_t::HUMAN}; + } else if (algorithm_type_t::VEHICLE_RETROGRADE == algor_type) { + return { + det_class_label_t::LARGE_CAR, det_class_label_t::MEDIUM_BUS, det_class_label_t::SMALL_CAR, + det_class_label_t::TRUCK, det_class_label_t::TRACTOR, + + // det_class_label_t::BICYCLE, + // det_class_label_t::MOTOCYCLE, + // det_class_label_t::TRICYCLE, + }; + } else { + return {}; + } +} + +bool is_valid_box(const int &cls, const algorithm_type_t &algor_type) { + return algor_type_to_det_label_set(algor_type).count(static_cast(cls)); +} + +bool is_valid_box(const box_t &box, const algorithm_type_t &algor_type, + const task_param_manager::algo_param_type_t_ *params_ptr = nullptr) { + if (!params_ptr) + return false; + + if (!snapshot_legal_inarea(params_ptr->basic_param->algor_valid_rect, box.left, box.top, box.right, box.bottom)) + return false; + + if (params_ptr->algor_param == nullptr) + return false; + + if (box.width() == 0 || box.height() == 0) + return false; + + using data_t = algor_config_param_retrograde_basic; + data_t *algor_params_ptr = (data_t *)(params_ptr->algor_param); + + if (box.score < algor_params_ptr->conf_threshold || box.width() < algor_params_ptr->minmum_width || + box.height() < algor_params_ptr->minmum_height) + return false; + + return is_valid_box(box.cls, algor_type); +} + +const task_param_manager::algo_param_type_t_ *get_task_param_ptr(const task_id_t &task_id, + const algorithm_type_t &algor_type, + task_param_manager *const task_param_manager) { + if (!task_param_manager) + return nullptr; + + auto &&algor_param_wrap = task_param_manager->get_task_other_param(task_id, algor_type); + if (!algor_param_wrap) + LOG_ERROR("{} is nullptr when get algor param from task_param", task_id); + return algor_param_wrap; +} + +std::vector task_id_to_algorithm_type_seq(const task_id_t &task_id, + task_param_manager *const task_param) { + std::vector seq; + auto &&algor_map = task_param->get_task_other_param(task_id); + for (auto iter = algor_map->begin(); iter != algor_map->end(); ++iter) { + // LOG_TRACE("task id is {} algor type is {}", task_id, int(iter->first)); + if (algor_type_list_.count(iter->first) > 0) + seq.emplace_back(iter->first); + } + return seq; // N(RVO) +} + +// ############################################################ // +// ! Class Member ! // +// ############################################################ // + +PedestrianVehicleRetrograde::PedestrianVehicleRetrograde() : task_param_manager_(nullptr) { + if (!task_param_manager_) + task_param_manager_ = task_param_manager::getInstance(); +} + +PedestrianVehicleRetrograde::~PedestrianVehicleRetrograde() = default; + +void PedestrianVehicleRetrograde::force_release_result(const task_id_t &task_id) { + for (auto iter = obj_to_alarm_boxes_.begin(); iter != obj_to_alarm_boxes_.end();) { + const auto &key = iter->first; + if (key.task_id == task_id) { + auto &values = iter->second; + for (auto &value : values) { + + if (value.roi_img_desc) { + VPCUtil::vpc_pic_desc_release(value.roi_img_desc); + } + + if (value.origin_img_desc) { + VPCUtil::vpc_pic_desc_release(value.origin_img_desc); + } + } + values.clear(); + iter = obj_to_alarm_boxes_.erase(iter); + } else { + ++iter; + } + } + //-221024 byzsh删除存储的轨迹信息-------------------------------------------------- + for (auto iter = obj_to_traces_.begin(); iter != obj_to_traces_.end();) { + const auto &key = iter->first; + if (key.task_id == task_id) { + auto &values = iter->second; + values.clear(); + iter = obj_to_traces_.erase(iter); + } else { + ++iter; + } + } + //------------------------------------------------------------------------- + + +} + +vector PedestrianVehicleRetrograde::get_results_by_id(const obj_key_t &id, bool do_erase) { + LOG_DEBUG("obj_to_alarm_boxes_ size:{}", obj_to_alarm_boxes_.size()); + + vector res; + + auto it = obj_to_alarm_boxes_.find(id); + if (it == obj_to_alarm_boxes_.end()) { + return res; + } + + res = it->second; + if (do_erase) + obj_to_alarm_boxes_.erase(id); + + if (obj_to_traces_.find(id) != obj_to_traces_.end()) { + obj_to_traces_.erase(id); //221009 !不删除会出现轨迹已消失但继续报警的情况,导致内存泄露 + } + + return res; +} + +vector PedestrianVehicleRetrograde::retrograde_aux() { + /** + * @brief 使用box的历史轨迹点和当前轨迹点结合超参中的线段进行逻辑判断出是否逆行. + * + */ + // LOG_TRACE("---> retrograde_aux called."); + + vector ret; + const unsigned minimum_boxes_num = MAX_TRACE_NUM - 1; + for (auto &obj_to_trace : obj_to_traces_) { + + auto &obj_key = obj_to_trace.first; + auto &task_id = obj_key.task_id; + auto &algor_type = obj_key.algor_type; + auto &&algor_type_seq = task_id_to_algorithm_type_seq(task_id, task_param_manager_); + // LOG_DEBUG("55555555555 {} {} {}", obj_key.task_id, obj_key.obj_id, obj_key.algor_type); + // LOG_TRACE("---> Ckpt-0"); + + const task_param_manager::algo_param_type_t_ *param_ptr_wrap = + get_task_param_ptr(task_id, algor_type, task_param_manager_); + if (!param_ptr_wrap) + continue; + + using param_type_t = algor_config_param_retrograde_basic; + const param_type_t *param_ptr = (param_type_t *)(param_ptr_wrap->algor_param); + + // LOG_TRACE("---> Ckpt-1"); + + auto &boxes = obj_to_trace.second; + const auto box_size = boxes.size(); + + if (box_size < minimum_boxes_num) + continue; + + // LOG_TRACE("---> Ckpt-2"); + + int point_x = boxes[box_size - minimum_boxes_num].cx(); + const int point_y = boxes[box_size - minimum_boxes_num].cy(); + + double k = 0.0, b = 0.0; + double dist_c = 0.0, dist_p = 0.0; // current point and prev(history) point. + + //! 使用超参中的点 连成的线水平还是垂直. + const bool is_horizontal_line = std::abs(int(param_ptr->px1 - param_ptr->px2)) >= 1; + if (is_horizontal_line) { + k = double(param_ptr->py1 - param_ptr->py2) / (param_ptr->px1 - param_ptr->px2); + b = double(param_ptr->px1 * param_ptr->py2 - param_ptr->px2 * param_ptr->py1) / (param_ptr->px1 - param_ptr->px2); + + dist_c = fabs(k * boxes[box_size - 1].cx() - boxes[box_size - 1].cy() + b) / sqrt(k * k + 1.0); + dist_p = fabs(k * point_x - point_y + b) / sqrt(k * k + 1.0); + } else { + k = 0.0; + b = double(param_ptr->px1); + + dist_c = fabs(boxes[box_size - 1].cx() - b); + dist_p = fabs(point_x - b); + } + + // 从历史轨迹点中找到当前点距离超过阈值的最小点. + bool flag = true; + if (abs(dist_c - dist_p) < MINIMUM_DISTANCE) { + flag = false; + int prev_idx = box_size - MAX_TRACE_NUM; + while (prev_idx >= 0) { + point_x = boxes[prev_idx].cx(); + dist_p = is_horizontal_line ? fabs(k * point_x - point_y + b) / sqrt(k * k + 1.0) : fabs(point_x - b); + if (abs(dist_c - dist_p) >= MINIMUM_DISTANCE) { + flag = true; + break; + } + --prev_idx; + } + } + + // LOG_TRACE("---> Ckpt-10"); + + if (!flag) + continue; + + // 当前轨迹点和历史轨迹点是否越线 + bool flag_c = is_horizontal_line ? (k * boxes[box_size - 1].cx() + b >= boxes[box_size - 1].cy()) + : (boxes[box_size - 1].cx() >= b); + bool flag_p = is_horizontal_line ? (k * point_x + b >= point_y) : (point_x >= b); + + /** + * PS. 拌线: 使用超参中的两点模拟的线段. + * 1. POSITIVE && is_horizontal_line: 方向点在拌线上方,正常方向为从下到上(↑),越界方向为从上到下(↓) + * 2. NEGATIVE && is_horizontal_line: 方向点在拌线下方,正常方向为从上到下(↓),越界方向为从下到上(↑) + * 3. POSITIVE && !is_horizontal_line: 方向点在拌线右边,正常方向为从左到右(→),越界方向为从右到左(←) + * 4. NEGATIVE && !is_horizontal_line: 方向点在拌线左边,正常方向为从右到左(←),越界方向为从左到右(→) + */ + bool is_alarmed = false; + if (direction_t::POSITIVE == static_cast(param_ptr->direction)) { + if (flag_c * flag_p == 1) { + if (dist_c <= dist_p) { + is_alarmed = true; + } + } else if (flag_c == 0 && flag_p == 1) { + is_alarmed = true; + } else if (flag_c == 0 && flag_p == 0) { + if (dist_p <= dist_c) { + is_alarmed = true; + } + } + } else { + if (flag_c == 0 && flag_p == 0) { + if (dist_c <= dist_p) { + is_alarmed = true; + } + } else if (flag_c == 1 && flag_p == 0) { + is_alarmed = true; + } else if (flag_c == 1 && flag_p == 1) { + if (dist_p <= dist_c) { + is_alarmed = true; + } + } + } + + if (is_alarmed) { + ret.emplace_back(obj_key); + } + } + return ret; +} + +bool PedestrianVehicleRetrograde::update_mstreams(const std::vector &tasks_id, vector vec_det_input_images, + const std::vector &det_results) { + //! check. + if (tasks_id.empty() || tasks_id.size() != det_results.size()) + return false; + + //! loop. + unsigned stream_idx = 0; + std::map task_id_to_stream_idx; + for (auto task_id_iter = tasks_id.begin(); task_id_iter != tasks_id.end(); ++task_id_iter, ++stream_idx) { + auto task_id = *task_id_iter; + task_id_to_stream_idx[task_id] = stream_idx; + auto &&algor_type_seq = task_id_to_algorithm_type_seq(task_id, task_param_manager_); + if (algor_type_seq.empty()) + continue; + + auto &det_result = det_results[stream_idx]; + for (unsigned box_idx = 0; box_idx < det_result.obj_count; ++box_idx) { + auto &box = det_result.obj[box_idx]; + // LOG_DEBUG("333333333333 {} {}", task_id, box.id); + box_t unique_box; + { + unique_box.id = box.id; + unique_box.cls = box.index; + + unique_box.top = box.top; + unique_box.left = box.left; + unique_box.right = box.right; + unique_box.bottom = box.bottom; + unique_box.score = box.confidence; + } + + //! loop algor + for (auto algor_type : algor_type_seq) { + obj_key_t obj_key{box.id, task_id, algor_type}; + if (!is_valid_box(unique_box, algor_type, get_task_param_ptr(task_id, algor_type, task_param_manager_))) + continue; + + if (obj_to_traces_.find(obj_key) == obj_to_traces_.end()) { + obj_to_traces_[obj_key].emplace_back(std::move(unique_box)); + } else { + auto &traces = obj_to_traces_[obj_key]; + traces.emplace_back(std::move(unique_box)); + if (traces.size() > MAX_TRACE_NUM) + traces.pop_front(); + } + } + } + } + + vector &&alarm_objs = retrograde_aux(); // 右值引用 + + VPCUtil* pVpcUtil = VPCUtil::getInstance(); + + for (const auto &obj_key : alarm_objs) { + auto &traces = obj_to_traces_[obj_key]; + + if (traces.empty()) + continue; + + if (obj_to_alarm_boxes_.find(obj_key) != obj_to_alarm_boxes_.end() && obj_to_alarm_boxes_[obj_key].size() >= 10) + continue; + + auto &trace = traces.at(0); + auto &&it = task_id_to_stream_idx.find(obj_key.task_id); + if (it == task_id_to_stream_idx.end()) + continue; + + const unsigned stream_idx = it->second; + auto &src_img = vec_det_input_images[stream_idx]; + + result_data_t result_data; + result_data.box = trace; + + // 原图 + vpc_img_info src_img_info = VPCUtil::vpc_devMem2vpcImg(src_img); + result_data.origin_img_desc = src_img_info.pic_desc; + + // 抠图 + int width = src_img->getWidth(); + int height = src_img->getHeight(); + video_object_info obj; + strcpy(obj.task_id, obj_key.task_id.c_str()); + obj.object_id = obj_key.obj_id; + obj.left = clip(trace.left, 0, width); + obj.top = clip(trace.top, 0, height); + obj.right = clip(trace.right, 0, width); + obj.bottom = clip(trace.bottom, 0, height); + + vpc_img_info img_info = pVpcUtil->crop(src_img, obj); + result_data.roi_img_desc = img_info.pic_desc; + + // 保存结果 + obj_to_alarm_boxes_[obj_key].emplace_back(std::move(result_data)); + } + + LOG_DEBUG("obj_to_alarm_boxes_ size: {}", obj_to_alarm_boxes_.size()); + + return true; +} + +} // namespace pedestrian_vehicle_retrograde +} // namespace ai_engine_module diff --git a/src/ai_engine_module/pedestrian_vehicle_retrograde.h b/src/ai_engine_module/pedestrian_vehicle_retrograde.h new file mode 100644 index 0000000..a042d5f --- /dev/null +++ b/src/ai_engine_module/pedestrian_vehicle_retrograde.h @@ -0,0 +1,59 @@ +/* + * File: pedestrian_vehicle_retrograde.hpp + * Created Date: Tuesday February 22nd 2022 + * Author: yangzilong (yangzilong@objecteye.com) + * Description: + * ----- + * Last Modified: Tuesday, 22nd February 2022 4:35:04 pm + * Modified By: yangzilong (yangzilong@objecteye.com>) + * ----- + * Copyright 2022 + */ + +#pragma once + +#include +#include +#include +#include "../util/vpc_util.h" +#include "../ai_platform/task_param_manager.h" +#include "ai_engine_header.h" + +class DeviceMemory; + +namespace ai_engine_module { +namespace pedestrian_vehicle_retrograde { + + +class PedestrianVehicleRetrograde { + /** + * @brief + * 1. move able + */ +public: + PedestrianVehicleRetrograde(); + + ~PedestrianVehicleRetrograde(); + + bool update_mstreams(const std::vector &tasks_id, vector vec_det_input_images, + const std::vector &det_result); + + vector get_results_by_id(const obj_key_t &id, bool do_erase = true); + + void force_release_result(const task_id_t &task_id); + + PedestrianVehicleRetrograde(const PedestrianVehicleRetrograde &) = delete; + PedestrianVehicleRetrograde &operator=(const PedestrianVehicleRetrograde &) = delete; + + PedestrianVehicleRetrograde(PedestrianVehicleRetrograde &&) = default; + PedestrianVehicleRetrograde &operator=(PedestrianVehicleRetrograde &&) = default; + +private: + vector retrograde_aux(); + + task_param_manager *task_param_manager_; + std::map > obj_to_alarm_boxes_; + std::map> obj_to_traces_; +}; +} // namespace pedestrian_vehicle_retrograde +} // namespace ai_engine_module diff --git a/src/ai_engine_module/pedestrian_vehicle_trespass.cpp b/src/ai_engine_module/pedestrian_vehicle_trespass.cpp new file mode 100644 index 0000000..ba78a92 --- /dev/null +++ b/src/ai_engine_module/pedestrian_vehicle_trespass.cpp @@ -0,0 +1,358 @@ +/* + * File: pedestrian_vehicle_trespass.cpp + * Created Date: Tuesday February 22nd 2022 + * Author: yangzilong (yangzilong@objecteye.com) + * Description: + * ----- + * Last Modified: Tuesday, 22nd February 2022 4:38:48 pm + * Modified By: yangzilong (yangzilong@objecteye.com>) + * ----- + * Copyright 2022 + */ + +#include "./pedestrian_vehicle_trespass.h" +#include "opencv2/imgproc/imgproc.hpp" +#include "opencv2/opencv.hpp" +#include +#include "../decoder/interface/DeviceMemory.hpp" +#include "../common/logger.hpp" +#include "../ai_platform/mvpt_process_assist.h" + + +namespace ai_engine_module { +namespace pedestrian_vehicle_trespass { + +#define MAX_TRACE_NUM 5 +#define MINIMUM_DISTANCE 10 + +static std::set algor_type_list_ = { + algorithm_type_t::PEDESTRIAN_TRESPASS, + algorithm_type_t::VEHICLE_TRESPASS, +}; + +// void show_algorthim_result(sy_img img, box_t cur_box, const obj_key_t &key); +// ############################################################ // +// ! Auxiliary Function ! // +// ############################################################ // + +std::set algor_type_to_det_label_set(const algorithm_type_t &algor_type) { + if (algorithm_type_t::PEDESTRIAN_TRESPASS == algor_type) { + return {det_class_label_t::HUMAN}; + } else if (algorithm_type_t::VEHICLE_TRESPASS == algor_type) { + return { + det_class_label_t::LARGE_CAR, det_class_label_t::MEDIUM_BUS, det_class_label_t::SMALL_CAR, + det_class_label_t::TRUCK, det_class_label_t::TRACTOR, + }; + } else { + return {}; + } +} + +/* 是否是有效目标框的辅助判断函数 */ +bool is_valid_box(const int &cls, const algorithm_type_t &algor_type) { + return algor_type_to_det_label_set(algor_type).count(static_cast(cls)); +} + +bool is_valid_box(const box_t &box, const algorithm_type_t &algor_type, + const task_param_manager::algo_param_type_t_ *params_ptr = nullptr) { + if (!params_ptr) + return false; + + if (!snapshot_legal_inarea(params_ptr->basic_param->algor_valid_rect, box.left, box.top, box.right, box.bottom)) + return false; + + if (params_ptr->algor_param == nullptr) + return false; + + if (box.width() == 0 || box.height() == 0) + return false; + + using data_t = algor_config_param_trespass_basic; + auto *algor_params_ptr = (data_t *)(params_ptr->algor_param); + + if (box.score < algor_params_ptr->conf_threshold || box.width() < algor_params_ptr->minmum_width || + box.height() < algor_params_ptr->minmum_height) + return false; + + return is_valid_box(box.cls, algor_type); +} + +/* 获取指定任务的算法配置参数 */ +const task_param_manager::algo_param_type_t_ *get_task_param_ptr(const task_id_t &task_id, + const algorithm_type_t &algor_type, + task_param_manager *const task_param_manager) { + if (!task_param_manager) + return nullptr; + + auto &&algor_param_wrap = task_param_manager->get_task_other_param(task_id, algor_type); + if (!algor_param_wrap) + LOG_ERROR("{} is nullptr when get algor param from task_param", task_id); + return algor_param_wrap; +} + +/* 按照目标任务 检索指定任务的算法配置列表 */ +std::vector task_id_to_algorithm_type_seq(const task_id_t &task_id, + task_param_manager *const task_param) { + std::vector seq; + auto &&algor_map = task_param->get_task_other_param(task_id); + for (auto iter = algor_map->begin(); iter != algor_map->end(); ++iter) { + // LOG_TRACE("task id is {} algor type is {}", task_id, int(iter->first)); + if (algor_type_list_.count(iter->first) > 0) + seq.emplace_back(iter->first); + } + return seq; // N(RVO) +} + +// ############################################################ // +// ! Class Member ! // +// ############################################################ // + +PedestrianVehicleTrespass::PedestrianVehicleTrespass() : task_param_manager_(nullptr) { + if (!task_param_manager_) + task_param_manager_ = task_param_manager::getInstance(); +} + +PedestrianVehicleTrespass::~PedestrianVehicleTrespass() = default; + +/* 根据目标id 获取该目标的算法分析结果 */ +vector PedestrianVehicleTrespass::get_results_by_id(const obj_key_t &id, bool do_erase) { + + vector res; + auto it = obj_to_alarm_boxes_.find(id); + if (it == obj_to_alarm_boxes_.end()) { + // printf("cant find %s\n", id.task_id.c_str()); + return res; + } + + res = it->second; + if (do_erase) + obj_to_alarm_boxes_.erase(id); + + return res; +} + +/* 目标在禁区内外的辅助判断函数 */ +bool PedestrianVehicleTrespass::in_rect_analysis(const obj_key_t &id, const box_t &cur_bos) { + int center_x = int((cur_bos.left + cur_bos.right) / 2.0); + int center_y = int((cur_bos.top + cur_bos.bottom) / 2.0); + + obj_key_t tmp_id = {0, id.task_id, id.algor_type}; + cv::Mat &dst_mat = trespass_regions[tmp_id]; + // printf("id=%d center: %d %d ", id.obj_id, center_x, center_y); + if (dst_mat.data[center_y * dst_mat.cols + center_x] && dst_mat.data[center_y * dst_mat.cols + center_x + 1] && + dst_mat.data[center_y * dst_mat.cols + center_x - 1] && dst_mat.data[(center_y + 1) * dst_mat.cols + center_x] && + dst_mat.data[(center_y - 1) * dst_mat.cols + center_x]) { + // printf(" true\n"); + return true; // 进入禁区 + } else { + // printf(" false\n"); + return false; // 未进入禁区 + } +} + +/* 根据用户输入的点 初始化禁区区域mask */ +void PedestrianVehicleTrespass::pedestrianvehicletrespass_init_region(const string &task_id, + const algorithm_type_t algor_type, + const int width, const int height) { + obj_key_t obj_key = {0, task_id, algor_type}; + + auto &&algor_type_seq = task_id_to_algorithm_type_seq(task_id, task_param_manager_); + + const task_param_manager::algo_param_type_t_ *param_ptr_wrap = + get_task_param_ptr(task_id, algor_type, task_param_manager_); + if (!param_ptr_wrap) { + printf("cant find %s algorthim params\n", task_id.c_str()); + return; + } + + using param_type_t = algor_config_param_trespass_basic; + const param_type_t *param_ptr = (param_type_t *)(param_ptr_wrap->algor_param); + + cv::Mat src(height, width, CV_8UC3); + src.setTo(0); + + std::vector contour; + contour.reserve(param_ptr->points_count); + // printf("region points count: %d\n", param_ptr->points_count); + for (int idx = 0; idx < param_ptr->points_count; idx++) { + contour.emplace_back(param_ptr->points[idx].x_, param_ptr->points[idx].y_); + // printf(" =point:%d x:%d y:%d\n", idx, param_ptr->points[idx].x_, param_ptr->points[idx].y_); + } + + std::vector> contours; + contours.push_back(contour); + + cv::polylines(src, contours, true, cv::Scalar(255, 255, 255), 2, 8); // 第2个参数可以采用contour或者contours,均可 + cv::fillPoly(src, contours, cv::Scalar(255, 255, 255)); // fillPoly函数的第二个参数是二维数组 + + cv::Mat &dst_mat = trespass_regions[obj_key]; + cv::cvtColor(src, trespass_regions[obj_key], cv::COLOR_BGR2GRAY); + cv::threshold(trespass_regions[obj_key], trespass_regions[obj_key], 100, 255, cv::THRESH_BINARY); + + if (false) { + cv::imwrite("ori.jpg", src); + cv::imwrite("region.jpg", trespass_regions[obj_key]); + } +} + +/* 非法闯入禁区的 算法判断函数 */ +bool PedestrianVehicleTrespass::update_mstreams(const std::vector &tasks_id, vector det_input_images, + const std::vector &det_results, + const vector> &delete_objs) { + //! check. + if (tasks_id.empty() || tasks_id.size() != det_results.size()) + return false; + + vector alarm_objs; + //! loop. + unsigned stream_idx = 0U; + std::map task_id_to_stream_idx; + for (auto task_id_iter = tasks_id.begin(); task_id_iter != tasks_id.end(); ++task_id_iter, ++stream_idx) { + const auto &task_id = *task_id_iter; + task_id_to_stream_idx[task_id] = stream_idx; + // printf("\nbegin judge: %s\n", task_id.c_str()); + + /* 判断该路任务 是否开启该算法 */ + auto &&algor_type_seq = task_id_to_algorithm_type_seq(task_id, task_param_manager_); + if (algor_type_seq.empty()) + continue; + + // 删除 已经删除的目标 + for (auto obj_idx : delete_objs[stream_idx]) { + for (auto algor_type : algor_type_seq) { + obj_key_t obj_key{obj_idx, task_id, algor_type}; + + if (obj_to_position_.find(obj_key) != obj_to_position_.end()) { + // printf("delete obj: %s %d\n", task_id.c_str(), obj_idx); + obj_to_position_.erase(obj_key); + } + } + + // delete obj + } + + /* 依次判断检测目标框 是否有非法闯入 判断逻辑:之前帧在禁区外 当前帧进入禁区 */ + auto &det_result = det_results[stream_idx]; + // printf("det count: %d\n", det_result.obj_count); + for (unsigned box_idx = 0; box_idx < det_result.obj_count; ++box_idx) { + auto &box = det_result.obj[box_idx]; + + box_t unique_box{}; + { + unique_box.id = box.id; + unique_box.cls = box.index; + + unique_box.top = box.top; + unique_box.left = box.left; + unique_box.right = box.right; + unique_box.bottom = box.bottom; + unique_box.score = box.confidence; + } + + //! loop algor + for (auto algor_type : algor_type_seq) { + obj_key_t obj_key{box.id, task_id, algor_type}; + + if (!is_valid_box(unique_box, algor_type, get_task_param_ptr(task_id, algor_type, task_param_manager_))) + obj_to_position_.erase(obj_key); // 如果不满足条件 非 合法框 依然删除 + + //! add or update. + if (!in_rect_analysis(obj_key, unique_box)) // 禁区外 + { + // printf("push in: %s %d\n", task_id.c_str(), box.id); + obj_to_position_[obj_key] = unique_box; + } + + else { // 进入禁区 + if (obj_to_position_.find(obj_key) != obj_to_position_.end()) // 之前在禁区外,可报警 + { + obj_to_position_[obj_key] = unique_box; + + alarm_objs.emplace_back(obj_key); + } + + } + } + } + } + + /* 针对需要报警的目标 缓存当前大图&抠图 等待报警返回 */ + VPCUtil* pVpcUtil = VPCUtil::getInstance(); + for (const auto &obj_key : alarm_objs) { + auto &&it = task_id_to_stream_idx.find(obj_key.task_id); + if (it == task_id_to_stream_idx.end()) + continue; + + // 221009 byzsh记录10条即可-------------------------------------------------------------------------------------- + if (obj_to_alarm_boxes_.find(obj_key) != obj_to_alarm_boxes_.end() && obj_to_alarm_boxes_[obj_key].size() >= 10) + continue; + //-------------------------------------------------------------------------------------------------------------- + const unsigned stream_idx = it->second; + auto &src_img = det_input_images[stream_idx]; + + result_data_t result_data; + result_data.box = obj_to_position_[obj_key]; + + // 原图 + vpc_img_info src_img_info = VPCUtil::vpc_devMem2vpcImg(src_img); + result_data.origin_img_desc = src_img_info.pic_desc; + + // 抠图 + int width = src_img->getWidth(); + int height = src_img->getHeight(); + video_object_info obj; + strcpy(obj.task_id, obj_key.task_id.c_str()); + // obj.task_id = obj_key.task_id.c_str(); + obj.object_id = obj_key.obj_id; + obj.left = clip(result_data.box.left, 0, width); + obj.top = clip(result_data.box.top, 0, height); + obj.right = clip(result_data.box.right, 0, width); + obj.bottom = clip(result_data.box.bottom, 0, height); + + vpc_img_info img_info = pVpcUtil->crop(src_img, obj); + result_data.roi_img_desc = img_info.pic_desc; + + obj_to_alarm_boxes_[obj_key].emplace_back(std::move(result_data)); + obj_to_position_.erase(obj_key); + } + + return true; +} + +/* 辅助函数 显示算法结果 */ +// void show_algorthim_result(sy_img img, box_t cur_box, const obj_key_t &key) { +// const unsigned size = img.c_ * img.h_ * img.w_; +// auto *img_data = new unsigned char[size]; + +// CHECK(cudaMemcpy(img_data, img.data_, size * sizeof(unsigned char), cudaMemcpyDeviceToHost)); +// cv::Mat show_image(img.h_, img.w_, CV_8UC3, img_data); + +// std::vector contour; +// /* +// contour.push_back(cv::Point(200, 200)); +// contour.push_back(cv::Point(600, 200)); +// contour.push_back(cv::Point(600, 500)); +// contour.push_back(cv::Point(200, 500)); +// */ +// contour.emplace_back(500, 500); +// contour.emplace_back(1500, 500); +// contour.emplace_back(1500, 900); +// contour.emplace_back(500, 900); + +// std::vector> contours; +// contours.push_back(contour); + +// cv::polylines(show_image, contours, true, cv::Scalar(255, 255, 255), 2, +// 8); // 第2个参数可以采用contour或者contours,均可 + +// cv::rectangle(show_image, cv::Point(cur_box.left, cur_box.top), cv::Point(cur_box.right, cur_box.bottom), +// cv::Scalar(0, 250, 0), 2, 8); + +// char filename[256]; +// sprintf(filename, "res_image/%s_%ld.jpg", key.task_id.c_str(), key.obj_id); +// cv::imwrite(filename, show_image); +// printf("\n *****ERROR finish save %s %ld\n", key.task_id.c_str(), key.obj_id); +// } + +} // namespace pedestrian_vehicle_trespass +} // namespace ai_engine_module diff --git a/src/ai_engine_module/pedestrian_vehicle_trespass.h b/src/ai_engine_module/pedestrian_vehicle_trespass.h new file mode 100644 index 0000000..fcca210 --- /dev/null +++ b/src/ai_engine_module/pedestrian_vehicle_trespass.h @@ -0,0 +1,60 @@ +/* + * File: pedestrian_vehicle_trespass.hpp + * Created Date: Tuesday February 22nd 2022 + * Author: yangzilong (yangzilong@objecteye.com) + * Description: + * ----- + * Last Modified: Tuesday, 22nd February 2022 4:35:04 pm + * Modified By: yangzilong (yangzilong@objecteye.com>) + * ----- + * Copyright 2022 + */ + +#pragma once +#include "opencv2/highgui/highgui.hpp" +#include +#include +#include +#include "../util/vpc_util.h" +#include "../ai_platform/task_param_manager.h" +#include "ai_engine_header.h" + +class DeviceMemory; + +namespace ai_engine_module { +namespace pedestrian_vehicle_trespass { + +class PedestrianVehicleTrespass { + /** + * @brief + * 1. move able + */ +public: + PedestrianVehicleTrespass(); + + ~PedestrianVehicleTrespass(); + + void pedestrianvehicletrespass_init_region(const string &task_id, const algorithm_type_t algor_type, const int width, + const int height); + + bool update_mstreams(const std::vector &tasks_id, vector det_input_images, + const std::vector &det_result, const vector> &delete_objs); + + vector get_results_by_id(const obj_key_t &id, bool do_erase = true); + + PedestrianVehicleTrespass(const PedestrianVehicleTrespass &) = delete; + PedestrianVehicleTrespass &operator=(const PedestrianVehicleTrespass &) = delete; + + PedestrianVehicleTrespass(PedestrianVehicleTrespass &&) = default; + PedestrianVehicleTrespass &operator=(PedestrianVehicleTrespass &&) = default; + +private: + bool in_rect_analysis(const obj_key_t &id, const box_t &cur_bos); + + task_param_manager *task_param_manager_; + std::map > obj_to_alarm_boxes_; + std::map obj_to_position_; // 保存物体上一帧的位置,基于非法闯入判断逻辑,上一帧在框外,下一帧闯入禁区 + std::map trespass_regions; +}; +} // namespace pedestrian_vehicle_trespass +} // namespace ai_engine_module diff --git a/src/ai_platform/MultiSourceProcess.cpp b/src/ai_platform/MultiSourceProcess.cpp index 8e51ed9..e44774d 100644 --- a/src/ai_platform/MultiSourceProcess.cpp +++ b/src/ai_platform/MultiSourceProcess.cpp @@ -17,6 +17,8 @@ #include "macro_definition.h" #include "SourceSingleton.hpp" +#include "../util/vpc_util.h" + // #define VEHICLE_MULTI_BOXES using namespace std; @@ -72,7 +74,6 @@ CMultiSourceProcess::CMultiSourceProcess(){ } CMultiSourceProcess::~CMultiSourceProcess(){ - vpc_util.release(); } int CMultiSourceProcess::InitAlgorthim(tsl_aiplatform_param vptParam){ @@ -103,7 +104,8 @@ int CMultiSourceProcess::InitAlgorthim(tsl_aiplatform_param vptParam){ m_snapshot_reprocessing = new snapshot_reprocessing(m_devId); m_save_snapshot_reprocessing = new save_snapshot_reprocessing(m_devId); - vpc_util.init(m_devId); + VPCUtil* pVpcUtil = VPCUtil::getInstance(); + pVpcUtil->init(m_devId); m_pAlgorthimThread = new thread([](void* arg) { CMultiSourceProcess* process = (CMultiSourceProcess*)arg ; @@ -499,23 +501,32 @@ int CMultiSourceProcess::algorthim_vpt(vector vec_gpuMem){ /* 一级检测器,内部已完成跟踪操作 */ vpt_process.process_gpu(vpt_interest_imgs.data(), vpt_interest_task_id, vptResult, deleteObjectID, unUsedResult); // do det & track. -// m_snapshot_reprocessing->screen_effective_snapshot(vptResult); + m_snapshot_reprocessing->screen_effective_snapshot(vptResult); + +#ifndef VEHICLE_MULTI_BOXES + /* 快照优选(内部可实现不同的快照优选策略) */ + m_snapshot_reprocessing->update_bestsnapshot(vec_vptMem, vptResult, deleteObjectID); + + /* for snapshot algorithm. 轨迹结束目标 做最后的结果返回(当前返回算法结果+快照保存路径)*/ + vehicle_snapshot(vpt_interest_task_id, deleteObjectID); +#else + algorithm_vehicle_relult(vec_vptMem, vptResult, deleteObjectID); + + send_locus_finished_msg(vpt_interest_task_id, deleteObjectID); +#endif -// #ifndef VEHICLE_MULTI_BOXES -// /* 快照优选(内部可实现不同的快照优选策略) */ -// m_snapshot_reprocessing->update_bestsnapshot(vec_vptMem, vptResult, deleteObjectID); +// #ifdef WITH_SECOND_PROCESS + /* for pedestrian safety det. 行人安全分析算法模块 */ + // algorthim_pedestrian_safety(vpt_interest_task_id, vpt_interest_imgs,vptResult); -// /* for snapshot algorithm. 轨迹结束目标 做最后的结果返回(当前返回算法结果+快照保存路径)*/ -// vehicle_snapshot(vpt_interest_task_id, deleteObjectID); -// #else -// algorithm_vehicle_relult(vec_vptMem, vptResult, deleteObjectID); + /* for retrograde & trespass algor 逆行&非法闯入算法模块 */ + algorthim_retrograde_trespass(vpt_interest_task_id, vec_vptMem, vptResult, deleteObjectID); -// send_locus_finished_msg(vpt_interest_task_id, deleteObjectID); // #endif -// if(vptResult.size() > 0){ -// cout << vptResult[0].obj_count<< endl; -// } + if(vptResult.size() > 0){ + cout << vptResult[0].obj_count<< endl; + } vptResult.clear(); unUsedResult.clear(); deleteObjectID.clear(); @@ -532,6 +543,7 @@ int CMultiSourceProcess::algorithm_vehicle_relult(vector vec_devM vector results = m_snapshot_reprocessing->get_vehicle_snapshot(vec_devMem, vptResult, skip_frame_); + VPCUtil* pVPCUtil = VPCUtil::getInstance(); for (auto &result : results) { if(result.objs.size() <= 0){ continue; @@ -551,13 +563,13 @@ int CMultiSourceProcess::algorithm_vehicle_relult(vector vec_devM ImgSaveInfo saveInfo; saveInfo.file_path = fpath_origin; - saveInfo.img_info = vpc_util.vpc_devMem2vpcImg(result.memPtr); + saveInfo.img_info = VPCUtil::vpc_devMem2vpcImg(result.memPtr); m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(saveInfo); - vector vec_obj_info_list = vpc_util.crop_batch(result.memPtr, result.objs); + vector vec_obj_info_list = pVPCUtil->crop_batch(result.memPtr, result.objs); if(vec_obj_info_list.size() != result.objs.size()){ LOG_ERROR("vpc_crop size error !"); - vpc_util.vpc_imgList_release(vec_obj_info_list); + VPCUtil::vpc_imgList_release(vec_obj_info_list); continue; } @@ -645,22 +657,73 @@ void CMultiSourceProcess::vehicle_snapshot(vector& vpt_interest_task_id, } /* 轨迹结束帧需要做的算法模块 */ -/* 轨迹结束帧需要做的算法模块 */ int CMultiSourceProcess::endframe_obj_process(const OBJ_KEY &obj_key, algorithm_type_t algor_type) { + + // 该路任务开启了抓拍功能 开始抓拍保存;若未开启抓拍,清空显存资源 + vehicle_locus_finished(obj_key); + + /* 开启行人&机动车逆行算法模块,获取该目标的算法分析结果 返回结果+快照 最后释放资源 */ + endframe_retrograde_obj_process(obj_key); +} + +int CMultiSourceProcess::endframe_retrograde_obj_process(const OBJ_KEY &obj_key) { + auto task_param_ptr = m_task_param_manager->get_task_algor_param(obj_key.video_id); + + /* 开启行人&机动车逆行算法模块,获取该目标的算法分析结果 返回结果+快照 最后释放资源 */ + if (task_param_ptr->human_algors.find(algorithm_type_t::PEDESTRIAN_RETROGRADE) != + task_param_ptr->human_algors.end()) + retrograde_alarm(obj_key, algorithm_type_t::PEDESTRIAN_RETROGRADE); + + if (task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_RETROGRADE) != + task_param_ptr->vehicle_algors.end()) + retrograde_alarm(obj_key, algorithm_type_t::VEHICLE_RETROGRADE); +} + +void CMultiSourceProcess::retrograde_alarm(const OBJ_KEY &obj_key, const algorithm_type_t &algor_type) { + + auto results = pedestrian_vehicle_retrograde_.get_results_by_id( + ai_engine_module::obj_key_t{obj_key.obj_id, obj_key.video_id, algor_type}); + + bool bRetroGrade = true ; + for (unsigned idx = 0; idx < results.size(); ++idx) { + auto &result = results[idx]; + + vpc_img_info src_img; + src_img.pic_desc = result.origin_img_desc; + src_img.task_id = obj_key.video_id; + + vpc_img_info roi_img; + roi_img.pic_desc = result.roi_img_desc; + roi_img.task_id = obj_key.video_id; + roi_img.object_id = obj_key.obj_id; + + if(bRetroGrade){ + auto &&json_str = helpers::gen_json::gen_retrograde_json(obj_key.video_id, obj_key.obj_id, result.box, algor_type); + save_snapshot_process(obj_key, algor_type, src_img, roi_img, idx, json_str); + } else { + VPCUtil::vpc_img_release(src_img); + VPCUtil::vpc_img_release(roi_img); + } + + if (bRetroGrade == true) { + bRetroGrade = false; + } + + } + +} + +void CMultiSourceProcess::vehicle_locus_finished(const OBJ_KEY obj_key) { auto task_param_ptr = m_task_param_manager->get_task_algor_param(obj_key.video_id); auto task_other_param_ptr = m_task_param_manager->get_task_other_param(obj_key.video_id); // 该路任务开启了抓拍功能 开始抓拍保存;若未开启抓拍,清空显存资源 - if ((task_param_ptr->human_algors.find(algorithm_type_t::HUMAN_SNAPSHOT) != task_param_ptr->human_algors.end() || - task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_SNAPSHOT) != task_param_ptr->vehicle_algors.end() || + if ((task_param_ptr->human_algors.find(algorithm_type_t::HUMAN_SNAPSHOT) == task_param_ptr->human_algors.end() && + task_param_ptr->vehicle_algors.find(algorithm_type_t::VEHICLE_SNAPSHOT) != task_param_ptr->vehicle_algors.end() && task_param_ptr->nonmotor_vehicle_algors.find(algorithm_type_t::NONMOTOR_VEHICLE_SNAPSHOT) != task_param_ptr->nonmotor_vehicle_algors.end())) { - - vehicle_locus_finished(obj_key); - + return; } -} -void CMultiSourceProcess::vehicle_locus_finished(const OBJ_KEY obj_key){ map _total_snapshot_info = m_snapshot_reprocessing->get_total_snapshot_info(); LOG_DEBUG("_total_snapshot_info size: {}", _total_snapshot_info.size()); @@ -780,4 +843,88 @@ void CMultiSourceProcess::timing_snapshot_thread(){ jpegUtil.jpeg_release(); LOG_INFO("timing_snapshot_thread end."); +} + +void CMultiSourceProcess::algorthim_retrograde_trespass(vector& vpt_interest_task_id, vector vpt_interest_imgs, + vector& vptResult ,vector>& deleteObjectID){ + vector interest_task_id; + vector interest_vpt_result; + vector interest_imgs; + + vector trespass_interest_task_id; + vector trespass_interest_vpt_result; + vector> trespass_interest_deleteobjs; + vector trespass_interest_imgs; + + int _idx = 0; + for (auto _task_id_iter = vpt_interest_task_id.begin(); _task_id_iter != vpt_interest_task_id.end(); + ++_task_id_iter, ++_idx) // loop task_id; + { + auto task_id = *_task_id_iter; + auto algor_map = m_task_param_manager->get_task_other_param(task_id); + + if (algor_map->find(algorithm_type_t::PEDESTRIAN_RETROGRADE) != algor_map->end() || + algor_map->find(algorithm_type_t::VEHICLE_RETROGRADE) != algor_map->end()) { + interest_task_id.emplace_back(task_id); + interest_imgs.emplace_back(vpt_interest_imgs[_idx]); + interest_vpt_result.emplace_back(vptResult[_idx]); + } + + if (algor_map->find(algorithm_type_t::PEDESTRIAN_TRESPASS) != algor_map->end() || + algor_map->find(algorithm_type_t::VEHICLE_TRESPASS) != algor_map->end()) { + trespass_interest_task_id.emplace_back(task_id); + trespass_interest_imgs.emplace_back(vpt_interest_imgs[_idx]); + trespass_interest_vpt_result.emplace_back(vptResult[_idx]); + trespass_interest_deleteobjs.emplace_back(deleteObjectID[_idx]); + } + } + + LOG_DEBUG("trespass_interest_vpt_result size: {}", trespass_interest_vpt_result.size()); + + if (!interest_imgs.empty()) + pedestrian_vehicle_retrograde_.update_mstreams(interest_task_id, interest_imgs, + interest_vpt_result); + + if (!trespass_interest_imgs.empty()) { + pedestrian_vehicle_trespass_.update_mstreams( + trespass_interest_task_id, trespass_interest_imgs, trespass_interest_vpt_result, + trespass_interest_deleteobjs); + } +} + +bool CMultiSourceProcess::save_snapshot_process(const OBJ_KEY &obj_key, const algorithm_type_t &algorithm_type, vpc_img_info src_img, vpc_img_info roi_img, + const long long id,const std::string &json_str) { + auto task_other_params = m_task_param_manager->get_task_other_param(obj_key.video_id); + const auto &algor_other_params = task_other_params->find(algorithm_type); + if (algor_other_params == task_other_params->end()) { + LOG_ERROR("task_id {} not found {} error", obj_key.video_id, int(algorithm_type)); + return false; + } + + const algor_basic_config_param_t *basic_param = algor_other_params->second->basic_param; + + std::string cur_timestamp_ms = std::to_string(helpers::timer::get_timestamp()); + const std::string fpath_origin = basic_param->result_folder + helpers::os::sep + obj_key.video_id + "_" + + std::to_string(obj_key.obj_id) + "_" + std::to_string(id) + "_" + cur_timestamp_ms + ".jpg"; + + ImgSaveInfo obj_save_info; + obj_save_info.file_path = fpath_origin; + obj_save_info.img_info = src_img; + m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(obj_save_info); + + { + // 抠图保存 + const std::string fpath_roi = basic_param->result_folder_little + helpers::os::sep + obj_key.video_id + "_" + + std::to_string(obj_key.obj_id) + "_" + std::to_string(id) + "_" + cur_timestamp_ms + ".jpg"; + + // 调用快照保存后处理模块 将快照保存 + ImgSaveInfo obj_save_info; + obj_save_info.file_path = fpath_roi; + obj_save_info.img_info = roi_img; + obj_save_info.json_str = json_str; + m_save_snapshot_reprocessing->reprocessing_process_wo_locus_async(obj_save_info); + } + + + return true; } \ No newline at end of file diff --git a/src/ai_platform/MultiSourceProcess.h b/src/ai_platform/MultiSourceProcess.h index 362b8a5..59c1567 100644 --- a/src/ai_platform/MultiSourceProcess.h +++ b/src/ai_platform/MultiSourceProcess.h @@ -5,7 +5,8 @@ #include "../ai_engine_module/VPTProcess.h" #include "../reprocessing_module/snapshot_reprocessing.h" #include "../reprocessing_module/save_snapshot_reprocessing.h" -#include "../util/vpc_util.h" +#include "../ai_engine_module/pedestrian_vehicle_retrograde.h" +#include "../ai_engine_module/pedestrian_vehicle_trespass.h" #include "../util/JpegUtil.h" #include @@ -48,6 +49,9 @@ public: private: // 算法相关 int algorthim_vpt(vector vec_gpuMem); + // 逆行&非法闯入算法模块 + void algorthim_retrograde_trespass(vector& vpt_interest_task_id, vector vpt_interest_imgs, + vector& vptResult ,vector>& deleteObjectID); private: // 工具处理函数 @@ -61,6 +65,12 @@ private: int endframe_obj_process(const OBJ_KEY &obj_key, algorithm_type_t algor_type); + bool save_snapshot_process(const OBJ_KEY &obj_key, const algorithm_type_t &algorithm_type, vpc_img_info img_info, vpc_img_info roi_img, + const long long id,const std::string &json_str); + + int endframe_retrograde_obj_process(const OBJ_KEY &obj_key); + void retrograde_alarm(const OBJ_KEY &obj_key, const algorithm_type_t &algor_type) ; + private: int m_devId; @@ -87,11 +97,14 @@ private: set m_total_snapshot_info_multi_object; mutex m_total_mutex; - VPCUtil vpc_util; JpegUtil jpegUtil; #ifdef POST_USE_RABBITMQ mq::Manager *mq_manager_{nullptr}; #endif + + ai_engine_module::pedestrian_vehicle_retrograde::PedestrianVehicleRetrograde pedestrian_vehicle_retrograde_; + ai_engine_module::pedestrian_vehicle_trespass::PedestrianVehicleTrespass pedestrian_vehicle_trespass_; + }; \ No newline at end of file diff --git a/src/demo/demo.cpp b/src/demo/demo.cpp index fcdae7c..864b769 100644 --- a/src/demo/demo.cpp +++ b/src/demo/demo.cpp @@ -478,9 +478,9 @@ static long long get_cur_time(){ string createTask(void *handle, std::vector algor_vec, int gi){ task_param tparam; // tparam.ipc_url = "rtsp://admin:ad123456@192.168.60.108:554/cam/realmonitor?channel=1&subtype=0"; - // tparam.ipc_url = "/home/cmhu/data/bayue.mp4"; - // tparam.ipc_url = "/home/cmhu/data/Street.uvf"; - tparam.ipc_url = "rtsp://admin:ad123456@192.168.60.165:554/cam/realmonitor?channel=1&subtype=0"; + // tparam.ipc_url = "/data1/cmhu/data/bayue.mp4"; + tparam.ipc_url = "/data1/cmhu/data/Street.uvf"; + // tparam.ipc_url = "rtsp://admin:ad123456@192.168.60.165:554/cam/realmonitor?channel=1&subtype=0"; tparam.algor_counts = algor_vec.size(); tparam.dec_type = 2; @@ -521,7 +521,7 @@ void test_gpu(int gpuID){ printf("Init Success\n"); } - std::vector algor_vec = {algorithm_type_t::FACE_SNAPSHOT, algorithm_type_t::HUMAN_SNAPSHOT,algorithm_type_t::ROAD_WORK_DET, + std::vector algor_vec = {algorithm_type_t::FACE_SNAPSHOT, algorithm_type_t::HUMAN_SNAPSHOT,algorithm_type_t::ROAD_WORK_DET, algorithm_type_t::PEDESTRIAN_RETROGRADE, algorithm_type_t::VEHICLE_RETROGRADE, algorithm_type_t::VEHICLE_SNAPSHOT, algorithm_type_t::NONMOTOR_VEHICLE_SNAPSHOT, algorithm_type_t::VIDEO_TIMING_SNAPSHOT}; string task_id = createTask(handle, algor_vec, 0 + gpuID * 10); diff --git a/src/reprocessing_module/save_snapshot_reprocessing.cpp b/src/reprocessing_module/save_snapshot_reprocessing.cpp index d4dbf96..1fff9d8 100644 --- a/src/reprocessing_module/save_snapshot_reprocessing.cpp +++ b/src/reprocessing_module/save_snapshot_reprocessing.cpp @@ -5,6 +5,7 @@ #include #include #include "../common/logger.hpp" +#include "../util/vpc_util.h" const bool DRAW_ON_IMG = false; @@ -46,7 +47,7 @@ void save_snapshot_reprocessing::save_snapshot_reprocessing_release() { waitforsave_img_queue.pop(); if(!cur_image.file_path.empty()){ - vpc_util.vpc_img_release(cur_image.img_info); + VPCUtil::vpc_img_release(cur_image.img_info); } } @@ -96,7 +97,7 @@ void save_snapshot_reprocessing::save_img_process() { if(!cur_image.file_path.empty()){ jpegUtil.jpeg_encode(cur_image.img_info.pic_desc, cur_image.file_path); } - vpc_util.vpc_img_release(cur_image.img_info); + VPCUtil::vpc_img_release(cur_image.img_info); #ifdef POST_USE_RABBITMQ if (callback_ != nullptr && cur_image.json_str.length() > 0) { diff --git a/src/reprocessing_module/save_snapshot_reprocessing.h b/src/reprocessing_module/save_snapshot_reprocessing.h index 1dc1dd6..38d160a 100644 --- a/src/reprocessing_module/save_snapshot_reprocessing.h +++ b/src/reprocessing_module/save_snapshot_reprocessing.h @@ -15,8 +15,8 @@ #include #include "../ai_platform/det_obj_header.h" -#include "../util/vpc_util.h" #include "../util/JpegUtil.h" +#include "../util/vpc_util.h" #ifdef POST_USE_RABBITMQ #include "post_reprocessing.hpp" @@ -60,7 +60,6 @@ private: int m_devId; JpegUtil jpegUtil; - VPCUtil vpc_util; #ifdef POST_USE_RABBITMQ callback_t callback_; diff --git a/src/reprocessing_module/snapshot_reprocessing.cpp b/src/reprocessing_module/snapshot_reprocessing.cpp index 0955203..16f9f67 100644 --- a/src/reprocessing_module/snapshot_reprocessing.cpp +++ b/src/reprocessing_module/snapshot_reprocessing.cpp @@ -12,13 +12,10 @@ snapshot_reprocessing::snapshot_reprocessing(int devId) algor_index_table["human"] = { (int)det_class_label_t::HUMAN }; algor_index_table["nonmotor_vehicle"] = { (int)det_class_label_t::BICYCLE, (int)det_class_label_t::MOTOCYCLE, (int)det_class_label_t::TRICYCLE }; algor_index_table["vehicle"] = { (int)det_class_label_t::SMALL_CAR, (int)det_class_label_t::LARGE_CAR, (int)det_class_label_t::TRUCK, (int)det_class_label_t::TRACTOR, (int)det_class_label_t::MEDIUM_BUS }; - - vpcUtil.init(devId); } snapshot_reprocessing::~snapshot_reprocessing() { - vpcUtil.release(); } static void box_expansion(video_object_info& obj_info, float expand_ratio, int frame_width, int frame_height){ @@ -175,6 +172,8 @@ void snapshot_reprocessing::update_bestsnapshot(vector vec_devMem map && algor_config_param = m_task_param_manager->get_task_algor_params(); map> && algor_param = m_task_param_manager->get_task_other_params(); + VPCUtil* pVpcUtil = VPCUtil::getInstance(); + for (size_t i = 0; i < vec_devMem.size(); i++){ onelevel_det_result det_result = ol_det_result[i]; @@ -316,15 +315,15 @@ void snapshot_reprocessing::update_bestsnapshot(vector vec_devMem } } - vector imgList = vpcUtil.crop_batch(memPtr, vec_obj_info); + vector imgList = pVpcUtil->crop_batch(memPtr, vec_obj_info); vec_obj_info.clear(); for (size_t i = 0; i < imgList.size(); i++) { vpc_img_info obj_info = imgList[i]; OBJ_KEY objKey = { obj_info.task_id, obj_info.object_id }; - vpcUtil.vpc_img_release(total_snapshot_info[objKey].snapShot); - total_snapshot_info[objKey].snapShot = vpcUtil.vpc_devMem2vpcImg(memPtr); - vpcUtil.vpc_img_release(total_snapshot_info[objKey].snapShotLittle); + VPCUtil::vpc_img_release(total_snapshot_info[objKey].snapShot); + total_snapshot_info[objKey].snapShot = VPCUtil::vpc_devMem2vpcImg(memPtr); + VPCUtil::vpc_img_release(total_snapshot_info[objKey].snapShotLittle); total_snapshot_info[objKey].snapShotLittle = obj_info; } imgList.clear(); @@ -360,8 +359,8 @@ void snapshot_reprocessing::release_finished_locus_snapshot(const string taskid, if (bRelease){ OBJ_VALUE ss = total_snapshot_info[cur_key]; - vpcUtil.vpc_img_release(ss.snapShot); - vpcUtil.vpc_img_release(ss.snapShotLittle); + VPCUtil::vpc_img_release(ss.snapShot); + VPCUtil::vpc_img_release(ss.snapShotLittle); } total_snapshot_info.erase(cur_key); @@ -373,8 +372,8 @@ void snapshot_reprocessing::release_finished_locus_snapshot(const string taskid, if (strcmp(ss->first.video_id.c_str(), taskid.c_str()) == 0) { if (bRelease){ - vpcUtil.vpc_img_release(ss->second.snapShot); - vpcUtil.vpc_img_release(ss->second.snapShotLittle); + VPCUtil::vpc_img_release(ss->second.snapShot); + VPCUtil::vpc_img_release(ss->second.snapShotLittle); } total_snapshot_info.erase(ss); } diff --git a/src/reprocessing_module/snapshot_reprocessing.h b/src/reprocessing_module/snapshot_reprocessing.h index 1e7aa16..836b45a 100644 --- a/src/reprocessing_module/snapshot_reprocessing.h +++ b/src/reprocessing_module/snapshot_reprocessing.h @@ -82,6 +82,4 @@ private: map> algor_index_table; task_param_manager *m_task_param_manager; - - VPCUtil vpcUtil; }; \ No newline at end of file diff --git a/src/util/vpc_util.cpp b/src/util/vpc_util.cpp index 2ff6789..3f82d34 100644 --- a/src/util/vpc_util.cpp +++ b/src/util/vpc_util.cpp @@ -28,15 +28,6 @@ void VPCUtil::release() ret = acldvppDestroyChannel(dvppChannelDesc_); ret = acldvppDestroyChannelDesc(dvppChannelDesc_); - if (stream_ != nullptr) { - ret = aclrtDestroyStream(stream_); - if (ret != ACL_SUCCESS) { - LOG_ERROR("destroy stream failed"); - } - stream_ = nullptr; - } - LOG_INFO("end to destroy stream"); - if (context_ != nullptr) { ret = aclrtDestroyContext(context_); if (ret != ACL_SUCCESS) { @@ -54,8 +45,17 @@ void VPCUtil::release() } -int VPCUtil::vpc_crop(acldvppPicDesc *input_pic_desc, video_object_info obj) -{ +vpc_img_info VPCUtil::crop(DeviceMemory *devMem, video_object_info obj) { + + acldvppPicDesc *vpcInputDesc_ = acldvppCreatePicDesc(); + acldvppSetPicDescData(vpcInputDesc_, devMem->getMem()); + acldvppSetPicDescFormat(vpcInputDesc_, PIXEL_FORMAT_YUV_SEMIPLANAR_420); + acldvppSetPicDescWidth(vpcInputDesc_, devMem->getWidth()); + acldvppSetPicDescHeight(vpcInputDesc_, devMem->getHeight()); + acldvppSetPicDescWidthStride(vpcInputDesc_, devMem->getWidthStride()); + acldvppSetPicDescHeightStride(vpcInputDesc_, devMem->getHeightStride()); + acldvppSetPicDescSize(vpcInputDesc_, devMem->getSize()); + const int orimodelInputWidth = obj.right - obj.left; // cur model shape is 224 * 224 const int orimodelInputHeight = obj.bottom - obj.top; const int modelInputLeft = obj.left; // cur model shape is 224 * 224 @@ -87,22 +87,39 @@ int VPCUtil::vpc_crop(acldvppPicDesc *input_pic_desc, video_object_info obj) acldvppSetPicDescHeightStride(vpcOutputDesc_, modelInputHeight); acldvppSetPicDescSize(vpcOutputDesc_, vpcOutBufferSize_); - int ret = acldvppVpcCropAsync(dvppChannelDesc_, input_pic_desc, vpcOutputDesc_, cropArea_, stream_); - + aclrtStream stream_; + aclrtCreateStream(&stream_); + int ret = acldvppVpcCropAsync(dvppChannelDesc_, vpcInputDesc_, vpcOutputDesc_, cropArea_, stream_); ret = aclrtSynchronizeStream(stream_); + if (stream_ != nullptr) { + aclrtDestroyStream(stream_); + } + + acldvppDestroyPicDesc(vpcInputDesc_); + /* DestroycropResource */ (void)acldvppDestroyRoiConfig(cropArea_); cropArea_ = nullptr; - (void)acldvppDestroyPicDesc(vpcOutputDesc_); - vpcOutputDesc_ = nullptr; - if (vpcOutBufferDev_ != nullptr) { - (void)acldvppFree(vpcOutBufferDev_); - vpcOutBufferDev_ = nullptr; - } + vpc_img_info img_info ; + img_info.pic_desc = vpcOutputDesc_; + img_info.object_id = obj.object_id; + img_info.task_id = obj.task_id; //该物体属于的任务ID号 + img_info.task_frame_count = obj.task_frame_count; //该物体当前出现的帧号 + img_info.index = obj.index; //该物体所属类别的编号 + img_info.confidence = obj.confidence; //该物体的置信度 + + + // (void)acldvppDestroyPicDesc(vpcOutputDesc_); + // vpcOutputDesc_ = nullptr; + + // if (vpcOutBufferDev_ != nullptr) { + // (void)acldvppFree(vpcOutBufferDev_); + // vpcOutBufferDev_ = nullptr; + // } - return 0; + return img_info; } int VPCUtil::init(int32_t devId){ @@ -112,7 +129,6 @@ int VPCUtil::init(int32_t devId){ /* 2.Run the management resource application, including Device, Context, Stream */ aclrtSetDevice(deviceId_); aclrtCreateContext(&context_, deviceId_); - aclrtCreateStream(&stream_); // channel 准备 dvppChannelDesc_ = acldvppCreateChannelDesc(); @@ -153,6 +169,10 @@ vector VPCUtil::crop_batch(DeviceMemory *devMem, vectorgetMem()); acldvppSetPicDescFormat(vpcInputDesc_, PIXEL_FORMAT_YUV_SEMIPLANAR_420); @@ -166,6 +186,7 @@ vector VPCUtil::crop_batch(DeviceMemory *devMem, vector vecOutPtr_; @@ -216,10 +237,17 @@ vector VPCUtil::crop_batch(DeviceMemory *devMem, vector VPCUtil::crop_batch(DeviceMemory *devMem, vector crop_batch(DeviceMemory *devMem, vector objs); + static void vpc_pic_desc_release(acldvppPicDesc* ); - void vpc_img_release(vpc_img_info ); + static void vpc_img_release(vpc_img_info ); - void vpc_imgList_release(vector& ); + static void vpc_imgList_release(vector& ); - void release(); + static vpc_img_info vpc_devMem2vpcImg(DeviceMemory *devMem); + +public: + vpc_img_info crop(DeviceMemory *devMem, video_object_info obj); - vpc_img_info vpc_devMem2vpcImg(DeviceMemory *devMem); + int init(int32_t devId); + + vector crop_batch(DeviceMemory *devMem, vector objs); + + void release(); private: int32_t deviceId_; aclrtContext context_; - aclrtStream stream_; acldvppChannelDesc *dvppChannelDesc_; };