From 5e69f8fca9d5597af539904e4b67bae1f9c2bfc8 Mon Sep 17 00:00:00 2001 From: cmhu <2657262686@qq.com> Date: Sat, 11 Jan 2025 23:44:21 +0800 Subject: [PATCH] 补充字段； 优化逆行处理 --- jni/VehicleNativeInterface.cpp | 19 +++++++++++++++---- src/PicAnalysis.cpp | 15 ++++++++------- src/ai_engine_module/RoadSegAnalysis.cpp | 1064 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- src/ai_engine_module/RoadSegAnalysis.h | 20 ++++++++++++++------ src/village_inc.h | 5 +++-- 5 files changed, 449 insertions(+), 674 deletions(-) diff --git a/jni/VehicleNativeInterface.cpp b/jni/VehicleNativeInterface.cpp index 9afc1e5..f2a4316 100644 --- a/jni/VehicleNativeInterface.cpp +++ b/jni/VehicleNativeInterface.cpp @@ -452,7 +452,7 @@ JNIEXPORT jint JNICALL Java_com_objecteye_nativeinterface_vehicle_VehicleNativeI jobject vehicle_pendant_det_res_vpd_res_syRect_rect = env->NewObject(cls_SyRect, mid_SyRect, one_pendant_res.rect.left_, one_pendant_res.rect.top_, one_pendant_res.rect.width_, one_pendant_res.rect.height_); - jobject vehiclePendantDetInfo = env->NewObject(cls_VehiclePendantDetInfo, mid_VehiclePendantDetInfo, vehicle_pendant_det_res_vpd_res_syRect_rect, one_pendant_res.index, one_pendant_res.confidence, one_pendant_res.driver_copilot_info); + jobject vehiclePendantDetInfo = env->NewObject(cls_VehiclePendantDetInfo, mid_VehiclePendantDetInfo, vehicle_pendant_det_res_vpd_res_syRect_rect, one_pendant_res.index, one_pendant_res.confidence, one_pendant_res.driver_copilot_info, one_pendant_res.iColor); env->SetObjectArrayElement(vehiclePendantDetInfoArray, m, vehiclePendantDetInfo); } @@ -530,7 +530,18 @@ JNIEXPORT jint JNICALL Java_com_objecteye_nativeinterface_vehicle_VehicleNativeI } jint type = vehicle_info.type; - jobject vehicleInfo = env->NewObject(cls_VehicleInfo, mid_VehicleInfo, vehicleDetectResult, vehicleWinDetectResult, vehicleBodyDetectResult, vehicleColorResult, vehicleRecgResult, vplateResult, vehiclePendantDetResult, vehicleIllegalDetResult, vehicleFeaResult,mtaResult, MannedResult, type ); + jint vpt_type = vehicle_info.vpt_type; + jint rainshed = vehicle_info.rainshed; //是否安装雨棚， 0 有雨棚 1 无雨棚 + jint truck_manned = vehicle_info.truck_manned; + jint motor_manned = vehicle_info.motor_manned; + jint human_upper_color = vehicle_info.human_upper_color; + jint human_lower_color = vehicle_info.human_lower_color; + jint reverse_driving = vehicle_info.reverse_driving; // 0 正常行驶 1 逆行 -1 未知 + jint phoning = vehicle_info.phoning; // 骑车打电话 + jint cross_line = vehicle_info.cross_line; // 压实线， 1 黄实线 2 白实线 -1 其他 + jint cross_diversion_line = vehicle_info.cross_diversion_line; // 压导流线 1 压了 -1 未压 + + jobject vehicleInfo = env->NewObject(cls_VehicleInfo, mid_VehicleInfo, vehicleDetectResult, vehicleWinDetectResult, vehicleBodyDetectResult, vehicleColorResult, vehicleRecgResult, vplateResult, vehiclePendantDetResult, vehicleIllegalDetResult, vehicleFeaResult,mtaResult, MannedResult, type, vpt_type, rainshed, truck_manned, motor_manned, human_upper_color, reverse_driving, phoning, cross_line, cross_diversion_line); env->SetObjectArrayElement(vehicleInfoArray, j, vehicleInfo); } @@ -566,12 +577,12 @@ JNIEXPORT void JNICALL Java_com_objecteye_nativeinterface_vehicle_VehicleNativeI /* * Class: com_objecteye_nativeinterface_vehicle_VehicleNativeInterface - * Method: va_get_version + * Method: get_village_pic_version * Signature: ()Ljava/lang/String; */ JNIEXPORT jstring JNICALL Java_com_objecteye_nativeinterface_vehicle_VehicleNativeInterface_getVersion(JNIEnv *env, jobject obj) { - const char *version = va_get_version(); + const char *version = get_village_pic_version(); return env->NewStringUTF(version); } diff --git a/src/PicAnalysis.cpp b/src/PicAnalysis.cpp index 032d4f0..c5caf89 100644 --- a/src/PicAnalysis.cpp +++ b/src/PicAnalysis.cpp @@ -222,8 +222,8 @@ vector PicAnalysis::analysis_img(vector vec_img){ sy_rect vehicle_rect = result_info.vehicle_body_detect_res.rect; - if (road_info.direct_mask.cols > 0 && (1 == shot_type || 0 == shot_type)) { - result_info.reverse_driving = m_road_seg_algorithm.check_reverse_driving(road_info.direct_mask, vehicle_rect, src.width, src.height, shot_type); + if (road_info.vec_direct.size() > 0 && (1 == shot_type || 0 == shot_type)) { + result_info.reverse_driving = m_road_seg_algorithm.check_reverse_driving(road_info.vec_direct, vehicle_rect, src.width, src.height, shot_type); } // 行人 @@ -250,14 +250,14 @@ vector PicAnalysis::analysis_img(vector vec_img){ // 车尾，判断是否 货车尾部货厢载人 if(result_info.vpt_type == 6 || result_info.vpt_type == 7){ if(result_info.manned_res.hs_count > 0){ - result_info.manned = 1; + result_info.truck_manned = 1; } } } else { if(result_info.vpt_type == 1 || result_info.vpt_type == 2 || result_info.vpt_type == 3){ if(result_info.manned_res.hs_count >= 3){ // 摩托车、三轮车载人 - result_info.manned = 1; + result_info.motor_manned = 1; } } } @@ -329,9 +329,10 @@ vector PicAnalysis::analysis_img(vector vec_img){ } std::vector vec_hcp_result = m_human_car_algorithm.detect(vec_motor_img); - if (vec_hcp_result.size() > 0) { - if (vec_hcp_result[11] == 0 || vec_hcp_result[11] == 1) { - result_info.reverse_driving = m_road_seg_algorithm.check_reverse_driving(road_info.direct_mask, vehicle_rect, src.width, src.height, vec_hcp_result[11]); + if (vec_hcp_result.size() > 0 && road_info.vec_direct.size() > 0) { + int head_or_tail = vec_hcp_result[11].type; + if (head_or_tail == 0 || head_or_tail == 1) { + result_info.reverse_driving = m_road_seg_algorithm.check_reverse_driving(road_info.vec_direct, vehicle_rect, src.width, src.height, head_or_tail); } } diff --git a/src/ai_engine_module/RoadSegAnalysis.cpp b/src/ai_engine_module/RoadSegAnalysis.cpp index c273589..6c751de 100644 --- a/src/ai_engine_module/RoadSegAnalysis.cpp +++ b/src/ai_engine_module/RoadSegAnalysis.cpp @@ -3,518 +3,6 @@ uint8_t seg_colors[][3] = { {0, 0, 0}, {0, 255, 255}, {128, 255, 0}, {255, 128, 0}, {128, 0, 255}, {255, 0, 128}, {0, 128, 255}, {0, 255, 128}, {128, 255, 255}}; uint8_t lane_colors[][3] = { {0, 0, 0}, {255, 0, 0}, {0, 255, 0}, {0, 0, 255}, {255, 255, 0}, {255, 0, 255}, {0, 255, 255}, {128, 255, 0}, {255, 128, 0}}; -void lanes_process(const rs_lane* lanes, int lane_count, std::vector, int>>& combined, float scale_w = 1.0, float scale_h = 1.0) { - std::vector > lanes_xys; - std::vector lanes_cls; - for (int i = 0; i < lane_count; i++) { - std::vector xys; - for (int j = 0; j < lanes[i].num_points; j++) { - int x = static_cast(lanes[i].points[j].x_ * scale_w); - int y = static_cast(lanes[i].points[j].y_ * scale_h); - if (x > 0 && y > 0) { - xys.emplace_back(x, y); - } - } - if (!xys.empty()) { - lanes_xys.push_back(xys); - lanes_cls.push_back(lanes[i].cls); - } - } - - for (size_t i = 0; i < lanes_xys.size(); ++i) { - combined.push_back(std::make_pair(lanes_xys[i], lanes_cls[i])); - } - if (!combined.empty()) { - //按车道线起点坐标排序,相应的类别顺序也会变化以保证标签对齐 - std::sort(combined.begin(), combined.end(), [](const std::pair, int>& a, const std::pair, int>& b) { - return a.first[0].x < b.first[0].x; - }); - } -} - -cv::Mat mask_to_rgb(cv::Mat img, cv::Mat mask) { - cv::Mat rgb = img.clone(); - int reg_cls = 9; - for (int i = 0; i < rgb.rows; i++) { - for (int j = 0; j < rgb.cols; j++) { - for (int k = 1; k < reg_cls; k++) { - if (mask.at(i,j) == k) { - rgb.at(i,j)[0] = seg_colors[k][0]; - rgb.at(i,j)[1] = seg_colors[k][1]; - rgb.at(i,j)[2] = seg_colors[k][2]; - } - } - } - } - return rgb; -} - -float contourArea(std::vector contour, cv::Point2f& center) { - cv::RotatedRect rect = cv::minAreaRect(contour); // 最小外接矩形 rect[0]中心点 rect[1]宽 高 rect[2]旋转角度 - center = rect.center; - return cv::contourArea(contour); -} - -int Mask2LanePoints(const cv::Mat& pred, std::vector>&lanes, std::vector& cats) { - std::vector labels = {9, 10, 11, 12, 13}; - for(auto cat: labels) { - cv::Mat b_masks, measure_labels, stats, centroids; - cv::compare(pred, cat, b_masks, cv::CMP_EQ); //将pred元素逐个和cat比较，相同255，不同0 - // cv::threshold(pred, b_masks, 126, 255, cv::THRESH_OTSU); //生成二值图 - // 连通域计算 b_masks:闭操作后的二值图像 measure_labels:和原图一样大的标记图 centroids:nccomps×2的矩阵,表示每个连通域的质心(x, y) - // stats:nccomps×5的矩阵 表示每个连通区域的外接矩形和面积(x, y, w, h, area)，索引0是背景信息 - int nccomps = cv::connectedComponentsWithStats (b_masks, measure_labels, stats, centroids, 4); //8 - for(int cv_measure_id = 1; cv_measure_id < nccomps; cv_measure_id++ ) { //跳过背景信息0 - cv::Mat cv_measure_mask; - cv::compare(measure_labels, cv_measure_id, cv_measure_mask, cv::CMP_EQ); - std::vector< std::vector< cv::Point> > contours; - cv::findContours(cv_measure_mask,contours,cv::noArray(),cv::RETR_TREE,cv::CHAIN_APPROX_SIMPLE); - std::vector contours_poly; - for (int j = 0; j < contours.size();j++) { - // float area = cv::contourArea(contours[j]); if (area < 60) continue; //30 - float area = cv::contourArea(contours[j]); if (area < 30) continue; //30 - cv::approxPolyDP(cv::Mat(contours[j]), contours_poly, 6, true); //减小epsilon提高拟合精度(需要调试，epsilon过小会使多边形不够简化，单条线变多条) - if (contours_poly.size() == 1) continue; - lanes.push_back(contours_poly); - cats.push_back(cat); - } - } - } - return 0; -} - -int findMainMaskV2(bool large_resolution, const cv::Mat& solve_masks, const std::vector& labels, std::vector> &poly_masks, std::vector ®ion_classes, std::vector> &main_masks, cv::Mat& new_mask, cv::Mat& new_mask_forlane) { - int h = 360, w = 640; - int seg_num_cls = 8; - int seg_num_seg = 4; - int seg_min_region_area = 512; //10 - // mask部分区域 - if (large_resolution) solve_masks(cv::Rect(0, 0, w, int(h * 0.14))) = 0; - else solve_masks(cv::Rect(0, 0, w, int(h * 0.22))) = 0; - solve_masks(cv::Rect(0, int(h * 0.95), w, int(h * 0.05))) = 0; - solve_masks(cv::Rect(0, 0, int(w * 0.02), h)) = 0; - solve_masks(cv::Rect(int(w * 0.95), 0, int(w * 0.05), h)) = 0; - - float hf_w = w*0.5; - std::vector> all_emerg_contours, all_com_contours, mid_com_contours, mid_zd_contours; - std::vector emerg_contours, mid_contours, com_contours, zd_contours; - float emerg_max_area = 0, max_area = 0, zd_max_area = 0; - - for(auto cat: labels) { - if (cat == 0 || cat > seg_num_seg) - continue; - - // std::cout << cat << std::endl; - cv::Mat b_masks, measure_labels, stats, centroids; - cv::Mat n_masks = cv::Mat_(h,w); - n_masks = cv::Scalar(255); - cv::compare(solve_masks, cat, b_masks, cv::CMP_EQ); //将solve_masks元素逐个和cat比较，相同255，不同0 - //连通域计算 - int nccomps = cv::connectedComponentsWithStats (b_masks, measure_labels, stats, centroids, 8); - for(int i = 1; i < nccomps; i++ ) { //跳过背景信息0 - //移除过小的区域，并将对应位置置为0 - if (stats.at(i, cv::CC_STAT_AREA) < seg_min_region_area) { - cv::Mat comparison_result; - cv::compare(measure_labels, cv::Scalar(i), comparison_result, cv::CMP_EQ); //相等为255不等为0 - n_masks.setTo(0, comparison_result); // 将comparison_result中非零区域置为0 - cv::bitwise_and(solve_masks,n_masks,solve_masks); - continue; - } - - double centr_x = centroids.at(i, 0); - double centr_y = centroids.at(i, 1); - - int region_class = cat; - cv::Mat region_mask; - cv::bitwise_and(measure_labels,n_masks,measure_labels);///// - cv::compare(measure_labels, i, region_mask, cv::CMP_EQ); - - // 闭运算，先膨胀再腐蚀，去除较暗部分 - cv::Mat kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(17, 5)); - cv::morphologyEx(region_mask, region_mask, cv::MORPH_CLOSE, kernel); - - std::vector< std::vector< cv::Point> > contours; - cv::findContours(region_mask,contours,cv::noArray(),cv::RETR_TREE,cv::CHAIN_APPROX_SIMPLE); - std::vector contours_poly; - - // 应急车道 - if(region_class == 2) { - for (auto contour : contours) { - cv::Point2f center; - float area_ = contourArea(contour, center); - if (area_ > emerg_max_area) { - emerg_max_area = area_; - emerg_contours = contour; - } - all_emerg_contours.push_back(contour); - } - } - - // 普通行车道 - if(region_class == 1) { - for (auto contour : contours) { - cv::Point2f center; - float area_ = contourArea(contour, center); - if (area_ > max_area) { - max_area = area_; - com_contours = contour; - } - // 取中点所在的连通域 - cv::Point2f point(int(w*0.5),int(h*0.95)-1); - double distance = cv::pointPolygonTest(contour, point, false); - if (distance >= 0) { - mid_contours = contour; - hf_w = center.x; - continue; - } - if (abs(center.x - hf_w) < 90) // 取中点所在区域左右90像素内的连通域 230307 - mid_com_contours.push_back(contour); - } - } - - // 导流线 - if(region_class == 3) { - for (auto contour : contours) { - cv::Point2f center; - float area_ = contourArea(contour, center); - if (area_ > zd_max_area) { - zd_max_area = area_; - zd_contours = contour; - } - // if (abs(center.x - w*0.5) < 200) // 取中点左右200像素内的连通域 - if (abs(center.x - w*0.5) < 250) // 取中点左右250像素内的连通域 - mid_zd_contours.push_back(contour); - } - } - - // 拟合各行驶区域 - for (int j = 0; j < contours.size();j++) { - cv::approxPolyDP(cv::Mat(contours[j]), contours_poly, 10, true); - if (contours_poly.size() <= 2) continue; - poly_masks.push_back(contours_poly); - region_classes.push_back(cat); - } - } - } - - // cv::Mat new_mask(h, w, CV_8UC3, cv::Scalar(0,0,0)); - if (!mid_contours.empty()) { - // cv::fillConvexPoly(new_mask, mid_contours, cv::Scalar(255,255,255)); //填充255 - cv::fillConvexPoly(new_mask, mid_contours, cv::Scalar(255)); //填充255 - cv::fillConvexPoly(new_mask_forlane, mid_contours, cv::Scalar(255)); //填充255 - std::vector contours_poly; - cv::approxPolyDP(cv::Mat(mid_contours), contours_poly, 10, true); - if (contours_poly.size() > 2) main_masks.push_back(contours_poly); - all_com_contours.push_back(mid_contours); - } - else { - // cv::fillConvexPoly(new_mask, com_contours, cv::Scalar(255,255,255)); //填充255 - cv::fillConvexPoly(new_mask, com_contours, cv::Scalar(255)); //填充255 - cv::fillConvexPoly(new_mask_forlane, com_contours, cv::Scalar(255)); //填充255 - std::vector contours_poly; - cv::approxPolyDP(cv::Mat(com_contours), contours_poly, 10, true); - if (contours_poly.size() > 2) main_masks.push_back(contours_poly); - all_com_contours.push_back(com_contours); - } - - for (auto contour_ : mid_com_contours) { - // cv::fillConvexPoly(new_mask, contour_, cv::Scalar(255,255,255)); //填充255 - cv::fillConvexPoly(new_mask, contour_, cv::Scalar(255)); //填充255 - cv::fillConvexPoly(new_mask_forlane, contour_, cv::Scalar(255)); //填充255 - std::vector contours_poly; - cv::approxPolyDP(cv::Mat(contour_), contours_poly, 10, true); - if (contours_poly.size() > 2) main_masks.push_back(contours_poly); - all_com_contours.push_back(contour_); - } - - if (!emerg_contours.empty() && !all_emerg_contours.empty()) { - if (all_emerg_contours.size() == 1) - // cv::fillConvexPoly(new_mask, emerg_contours, cv::Scalar(255,255,255)); - cv::fillConvexPoly(new_mask, emerg_contours, cv::Scalar(255)); //填充255 - else { - float sum_centerx = 0, sum_centery = 0; - float length_emerg_com = 9999999; - // 计算选中行车道的平均中心点 - for (auto contour_ : all_com_contours) { - cv::Point2f center; - float area_ = contourArea(contour_, center); - sum_centerx += center.x; - sum_centery += center.y; - } - float ave_centerx = sum_centerx / all_com_contours.size(); - float ave_centery = sum_centery / all_com_contours.size(); - // 选择距离最近的应急车道且面积不小于最大应急车道的一半 - for (auto econtour_ : all_emerg_contours) { - cv::Point2f center; - float area_ = contourArea(econtour_, center); - float tmp_length = sqrt(pow((ave_centerx-center.x),2.0) + pow((ave_centery-center.y),2.0)); - if (tmp_length < length_emerg_com and area_ > 0.7*emerg_max_area) { - length_emerg_com = tmp_length; - emerg_contours = econtour_; - } - } - // cv::fillConvexPoly(new_mask, emerg_contours, cv::Scalar(255,255,255)); - cv::fillConvexPoly(new_mask, emerg_contours, cv::Scalar(255)); //填充255 - } - } - - if (!mid_zd_contours.empty()) { - for (auto contour_ : mid_zd_contours) - // cv::fillConvexPoly(new_mask, contour_, cv::Scalar(255,255,255)); //填充255 - cv::fillConvexPoly(new_mask, contour_, cv::Scalar(255)); //填充255 - } - - // 膨胀腐蚀，去除行车道和应急车道间的空洞 - cv::Mat kernel_ = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(17, 5)); - cv::morphologyEx(new_mask, new_mask, cv::MORPH_CLOSE, kernel_); - cv::morphologyEx(new_mask_forlane, new_mask_forlane, cv::MORPH_CLOSE, kernel_); - - return 0; -} - - -cv::Mat seg_post_process(bool large_resolution, unsigned char *seg_array, std::vector, int>> combined, std::vector> &poly_masks, std::vector ®ion_classes, std::vector> &lanes, std::vector &cats, std::map &x_sort, cv::Mat &background_mask) { - std::vector pred_cls; - int h = 360, w = 640; - cv::Mat lanes_masks = cv::Mat_(h,w); //正常分割结果 - cv::Mat mask_rmlane = cv::Mat_(h,w); //车道线区域置为背景 - cv::Mat solve_masks = cv::Mat_(h,w); //计算主行驶区域用(虚线及减速标线归入行车道) - int step_size = h*w; - int seg_min_region_area = 512; //1024 - - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; ++j) { - int max_cls = seg_array[(i * w + j)]; - lanes_masks.ptr(i)[j] = max_cls; - pred_cls.push_back(max_cls); - mask_rmlane.ptr(i)[j] = max_cls; - solve_masks.ptr(i)[j] = max_cls; - } - } - - for (const auto& lane_info : combined) { - const auto& xys = lane_info.first; - int cls = lane_info.second; - if (cls == 1) cls = 9; /*黄实线*/ if (cls == 2) cls = 10; /*白实线*/ - if (cls == 3) cls = 11; /*虚线*/ if (cls == 4) cls = 12; /*黄虚线*/ - if (cls == 5) cls = 13; /*车道中线*/ - for (size_t i = 1; i < xys.size(); ++i) { - cv::line(lanes_masks, xys[i - 1], xys[i], cls, 4); //绘制车道线用于求连通域 - if (cls == 11 || cls == 12) cls = 1; //求主行驶区域时将虚线归入行车道 - cv::line(solve_masks, xys[i - 1], xys[i], cls, 4); - - } - } - - // 求背景区域--mask车道区域，场景变化用 - cv::compare(lanes_masks, 0, background_mask, cv::CMP_EQ); //将lanes_masks元素逐个和0比较，相同255，不同0 -#if 0 - //--mask远处区域------------------------------------------------ - cv::Mat mask_black = mask_rmlane.clone(); - if (large_resolution) mask_black(cv::Rect(0, 0, w, int(h * 0.14))) = 0; - else mask_black(cv::Rect(0, 0, w, int(h * 0.22))) = 0; - mask_black(cv::Rect(0, int(h * 0.95), w, int(h * 0.05))) = 0; - mask_black(cv::Rect(0, 0, int(w * 0.02), h)) = 0; - mask_black(cv::Rect(int(w * 0.95), 0, int(w * 0.05), h)) = 0; - mask_rmlane = mask_black; - //------------------------------------------------------------------------- -#endif - //2.去重获取预测到的类别 - std::vector labels(pred_cls); - std::sort(labels.begin(),labels.end()); - labels.erase(std::unique(labels.begin(),labels.end()),labels.end()); - - //4.求车道线区域 - int flag = Mask2LanePoints(lanes_masks, lanes, cats); ///////////////////////// 车道线如何与mask结合 - - //5.求道路区域 - int count = 0; - for(auto cat: labels) { - // std::cout << cat << std::endl; - cv::Mat b_masks, measure_labels, stats, centroids; - cv::Mat n_masks = cv::Mat_(h,w); - n_masks = cv::Scalar(255); - cv::compare(mask_rmlane, cat, b_masks, cv::CMP_EQ); //将mask_rmlane元素逐个和cat比较，相同255，不同0 - //连通域计算 - int nccomps = cv::connectedComponentsWithStats (b_masks, measure_labels, stats, centroids, 8); - for(int i = 1; i < nccomps; i++ ) { //跳过背景信息0 - //移除过小的区域，并将对应位置置为0 - if (stats.at(i, cv::CC_STAT_AREA) < seg_min_region_area) { - cv::Mat comparison_result; - cv::compare(measure_labels, cv::Scalar(i), comparison_result, cv::CMP_EQ); //相等为255不等为0 - n_masks.setTo(0, comparison_result); // 将comparison_result中非零区域置为0 - - cv::bitwise_and(mask_rmlane,n_masks,mask_rmlane); - continue; - } - - double centr_x = centroids.at(i, 0); - double centr_y = centroids.at(i, 1); - - int region_class = cat; - // printf("region_class: %d\n", region_class); - cv::Mat region_mask; - cv::bitwise_and(measure_labels,n_masks,measure_labels);///// - cv::compare(measure_labels, i, region_mask, cv::CMP_EQ); - - std::vector< std::vector< cv::Point> > contours; - cv::findContours(region_mask,contours,cv::noArray(),cv::RETR_TREE,cv::CHAIN_APPROX_SIMPLE); - std::vector contours_poly; - for (int j = 0; j < contours.size();j++) { - cv::approxPolyDP(cv::Mat(contours[j]), contours_poly, 10, true); - if (contours_poly.size() <= 2) continue; - poly_masks.push_back(contours_poly); - region_classes.push_back(cat); - if (x_sort.count(centr_x)) centr_x += 0.0001; - x_sort.insert(std::make_pair(centr_x, count)); - ++ count; - // for (auto iter: contours_poly) { - // std::cout << "contour " << iter << " " << cat << std::endl; - // } - - } - } - } - -#if 0 - //6.draw lanes - for (int i = 0; i < lanes.size(); ++i) { - int thickness = 4; - for (int j = 0; j < lanes[i].size()-1; ++j) { - cv::line(mask_rmlane, lanes[i][j], lanes[i][j+1], cats[i], thickness); - // std::cout << lanes[i][j] << " " << lanes[i][j+1] << " " << cats[i] << std::endl; - } - } -#endif - return mask_rmlane; -} - - - -cv::Mat direct_post_process(bool large_resolution, unsigned char *direct_array, std::vector> &poly_masks, std::vector ®ion_classes) { - std::vector pred_cls; - int h = 360, w = 640, step_size = h*w, seg_min_region_area = 512; - cv::Mat mask_rmlane = cv::Mat_(h,w); //车道线区域置为背景 - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; ++j) { - int max_cls = direct_array[(i * w + j)]; - pred_cls.push_back(max_cls); - mask_rmlane.ptr(i)[j] = max_cls; - } - } - -#if 1 /*mask远处区域*/ - cv::Mat mask_black = mask_rmlane.clone(); - if (large_resolution) mask_black(cv::Rect(0, 0, w, int(h * 0.14))) = 0; - else mask_black(cv::Rect(0, 0, w, int(h * 0.22))) = 0; - mask_black(cv::Rect(0, int(h * 0.95), w, int(h * 0.05))) = 0; - mask_black(cv::Rect(0, 0, int(w * 0.02), h)) = 0; - mask_black(cv::Rect(int(w * 0.95), 0, int(w * 0.05), h)) = 0; - mask_rmlane = mask_black; -#endif - //去重获取预测到的类别 - std::vector labels(pred_cls); - std::sort(labels.begin(),labels.end()); - labels.erase(std::unique(labels.begin(),labels.end()),labels.end()); - - //求道路区域 - for(auto cat: labels) { - cv::Mat b_masks, measure_labels, stats, centroids; - cv::Mat n_masks = cv::Mat_(h,w); n_masks = cv::Scalar(255); - cv::compare(mask_rmlane, cat, b_masks, cv::CMP_EQ); //将mask_rmlane元素逐个和cat比较，相同255，不同0 - //连通域计算 - int nccomps = cv::connectedComponentsWithStats (b_masks, measure_labels, stats, centroids, 8); - for(int i = 1; i < nccomps; i++ ) { //跳过背景信息0 - if (stats.at(i, cv::CC_STAT_AREA) < seg_min_region_area) { //移除过小的区域，并将对应位置置为0 - cv::Mat comparison_result; - cv::compare(measure_labels, cv::Scalar(i), comparison_result, cv::CMP_EQ); //相等为255不等为0 - n_masks.setTo(0, comparison_result); // 将comparison_result中非零区域置为0 - cv::bitwise_and(mask_rmlane,n_masks,mask_rmlane); - continue; - } - - cv::Mat region_mask; - cv::bitwise_and(measure_labels,n_masks,measure_labels);///// - cv::compare(measure_labels, i, region_mask, cv::CMP_EQ); - - std::vector> contours; - cv::findContours(region_mask,contours,cv::noArray(),cv::RETR_TREE,cv::CHAIN_APPROX_SIMPLE); - std::vector contours_poly; - for (int j = 0; j < contours.size();j++) { - cv::approxPolyDP(cv::Mat(contours[j]), contours_poly, 10, true); - if (contours_poly.size() <= 2) continue; - poly_masks.push_back(contours_poly); - region_classes.push_back(cat); - } - } - } - return mask_rmlane; -} - - -// 合并来去区域 -cv::Mat merge_direct_process(bool large_resolution, unsigned char *direct_array, std::vector> &poly_masks, std::vector ®ion_classes) { - std::vector pred_cls; - int h = 360, w = 640, step_size = h*w, seg_min_region_area = 512; - cv::Mat mask_rmlane = cv::Mat_(h,w); //车道线区域置为背景 - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; ++j) { - int max_cls = direct_array[(i * w + j)]; - if (max_cls == 1 || max_cls == 2) max_cls = 5; // 来去区域合并的类别为5 - if (max_cls == 3 || max_cls == 4) max_cls = 6; // 近远区域合并的类别为6 - pred_cls.push_back(max_cls); - mask_rmlane.ptr(i)[j] = max_cls; - } - } - -#if 0 /*mask远处区域*/ - cv::Mat mask_black = mask_rmlane.clone(); - if (large_resolution) mask_black(cv::Rect(0, 0, w, int(h * 0.14))) = 0; - else mask_black(cv::Rect(0, 0, w, int(h * 0.22))) = 0; - mask_black(cv::Rect(0, int(h * 0.95), w, int(h * 0.05))) = 0; - mask_black(cv::Rect(0, 0, int(w * 0.02), h)) = 0; - mask_black(cv::Rect(int(w * 0.95), 0, int(w * 0.05), h)) = 0; - mask_rmlane = mask_black; -#endif - //去重获取预测到的类别 - std::vector labels(pred_cls); - std::sort(labels.begin(),labels.end()); - labels.erase(std::unique(labels.begin(),labels.end()),labels.end()); - - //求道路区域 - for(auto cat: labels) { - cv::Mat b_masks, measure_labels, stats, centroids; - cv::Mat n_masks = cv::Mat_(h,w); n_masks = cv::Scalar(255); - cv::compare(mask_rmlane, cat, b_masks, cv::CMP_EQ); //将mask_rmlane元素逐个和cat比较，相同255，不同0 - //连通域计算 - int nccomps = cv::connectedComponentsWithStats (b_masks, measure_labels, stats, centroids, 8); - for(int i = 1; i < nccomps; i++ ) { //跳过背景信息0 - if (stats.at(i, cv::CC_STAT_AREA) < seg_min_region_area) { //移除过小的区域，并将对应位置置为0 - cv::Mat comparison_result; - cv::compare(measure_labels, cv::Scalar(i), comparison_result, cv::CMP_EQ); //相等为255不等为0 - n_masks.setTo(0, comparison_result); // 将comparison_result中非零区域置为0 - cv::bitwise_and(mask_rmlane,n_masks,mask_rmlane); - continue; - } - - cv::Mat region_mask; - cv::bitwise_and(measure_labels,n_masks,measure_labels);///// - cv::compare(measure_labels, i, region_mask, cv::CMP_EQ); - - std::vector> contours; - cv::findContours(region_mask,contours,cv::noArray(),cv::RETR_TREE,cv::CHAIN_APPROX_SIMPLE); - std::vector contours_poly; - for (int j = 0; j < contours.size();j++) { - cv::approxPolyDP(cv::Mat(contours[j]), contours_poly, 10, true); - if (contours_poly.size() <= 2) continue; - poly_masks.push_back(contours_poly); - region_classes.push_back(cat); - } - } - } - return mask_rmlane; -} - RoadSegAnalysis::RoadSegAnalysis(/* args */) { } @@ -573,34 +61,91 @@ std::vector RoadSegAnalysis::detect(vector vec_img){ for (int b = 0; b < batchsize; b++) { auto one_result = results[b]; - RoadInfo one_road; - for (size_t i = 0; i < one_result.lane_count; i++) - { - auto one_lane = one_result.reg_array[i]; - LineInfo info; - for (size_t j = 0; j < one_lane.num_points; j++) - { - info.vec_pt.push_back(one_lane.points[j]); - } - info.line_type = one_lane.cls; - one_road.vec_line.push_back(info); - } + RoadInfo one_road = parse_road(one_result, vec_img[b]); + one_road.vec_direct = parse_direct(one_result, vec_img[b]); + vec_road.push_back(one_road); + } + + } while (0); + + for (int b = 0; b < batchsize; b++) { + delete[] results[b].seg_array; + results[b].seg_array = NULL; + delete[] results[b].direct_seg; + results[b].direct_seg = NULL; + } + + return vec_road; +} + +RoadInfo RoadSegAnalysis::parse_road(rs_result one_result, sy_img src) +{ + RoadInfo one_road; + + std::vector, int>> combined; + lanes_process(one_result.reg_array, one_result.lane_count, combined); + std::vector> poly_masks, lanes; + std::vector region_classes, cats; + std::map x_sort; + bool large_resolution = false; + if (src.w_ > 1920) large_resolution = true; + + cv::Mat seg_output = seg_post_process(large_resolution, one_result.seg_array, combined, poly_masks, region_classes, lanes, cats, x_sort); //m_masks:mask前的结果 poly_masks后的结果 - one_road.vec_road = parse_seg(one_result, vec_img[b]); - one_road.direct_mask = parse_direct(one_result, vec_img[b]); + for (int i = 0; i < lanes.size(); ++i) { + std::vector points = lanes[i]; + LineInfo info; + for (size_t j = 0; j < points.size(); j++) + { + sy_point pt; + pt.x_ = points[j].x; + pt.y_ = points[j].y; - post_direct(one_result, vec_img[b]); - - vec_road.push_back(one_road); + info.vec_pt.push_back(pt); } + info.line_type = cats[i]; + one_road.vec_line.push_back(info); + } + + for (int i = 0; i < poly_masks.size(); ++i) { + std::vector points = poly_masks[i]; + SegInfo seg_info; + for (int j = 0; j < points.size(); ++j) { + sy_point pt; + pt.x_ = points[j].x; + pt.y_ = points[j].y; - } while (0); + seg_info.vec_pt.push_back(pt); + } + seg_info.seg_type = region_classes[i]; + one_road.vec_road.push_back(seg_info); + } +} - for (int b = 0; b < batchsize; b++) { - delete[] results[b].seg_array; - results[b].seg_array = NULL; - delete[] results[b].direct_seg; - results[b].direct_seg = NULL; +std::vector RoadSegAnalysis::parse_direct(rs_result one_result, sy_img src) { + std::vector, int>> combined; + lanes_process(one_result.reg_array, one_result.lane_count, combined); + std::vector> poly_masks, lanes; + std::vector region_classes, cats; + std::map x_sort; + bool large_resolution = false; + if (src.w_ > 1920) large_resolution = true; + + cv::Mat seg_output = seg_post_process(large_resolution, one_result.direct_seg, combined, poly_masks, region_classes, lanes, cats, x_sort); //m_masks:mask前的结果 poly_masks后的结果 + + std::vector vec_road; + for (int i = 0; i < poly_masks.size(); ++i) { + std::vector points = poly_masks[i]; + SegInfo seg_info; + for (int j = 0; j < points.size(); ++j) { + sy_point pt; + pt.x_ = points[j].x; + pt.y_ = points[j].y; + + seg_info.vec_pt.push_back(pt); + } + seg_info.seg_type = region_classes[i]; + vec_road.push_back(seg_info); } return vec_road; @@ -622,95 +167,81 @@ int RoadSegAnalysis::release() { return SY_SUCCESS; } -std::vector RoadSegAnalysis::parse_seg(rs_result one_result, sy_img src) { - int w = 640; - int h = 360; - float alpha = 0.75; - cv::Mat overlayed_img(cv::Size(w,h), CV_8UC3, cv::Scalar(0, 0, 0)); - // 将车道线标签转换为彩色图像 - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; ++j) { - int idx = one_result.seg_array[(i * w + j)]; - overlayed_img.at(i, j)[0] = seg_colors[idx][0]; // R通道; - overlayed_img.at(i, j)[1] = seg_colors[idx][1]; // G通道 - overlayed_img.at(i, j)[2] = seg_colors[idx][2]; // B通道 - } - } +// head_or_tail 0：车头 1：车尾 +int RoadSegAnalysis::check_reverse_driving(std::vector& vec_direct, sy_rect src_rc, int src_width, int src_height, int head_or_tail) { - cv::resize(overlayed_img, overlayed_img, cv::Size(src.w_,src.h_), 0, 0, cv::INTER_LINEAR); - // 将原始图像和彩色车道线图进行混合 - // cv::addWeighted(cvImg, alpha, overlayed_img, 1 - alpha, 0, overlayed_img); + float scale_w = 640.0 / src_width; + float scale_h = 360.0 / src_height; - float scale_w = src.w_ / 640.0; - float scale_h = src.h_ / 360.0; - std::vector, int>> combined; - lanes_process(one_result.reg_array, one_result.lane_count, combined, scale_w, scale_h); + sy_rect rc; + rc.left_ = src_rc.left_ * scale_w; + rc.width_ = src_rc.width_ * scale_w; + rc.top_ = src_rc.top_ * scale_h; + rc.height_ = src_rc.height_ * scale_h; - std::vector vec_seg; - for (const auto& lane_info : combined) { - const auto& xys = lane_info.first; - int cls = lane_info.second; - SegInfo info; - info.seg_type = cls; - // cv::Scalar color(lane_colors[cls][0],lane_colors[cls][1],lane_colors[cls][2]); - for (size_t i = 0; i < xys.size(); ++i) { - // cv::line(overlayed_img, xys[i - 1], xys[i], color, 4); - sy_point pt; - pt.x_ = xys[i].x; - pt.y_ = xys[i].y; - info.vec_pt.push_back(pt); + std::vector polygon_pts; + cv::Point pt_lt; + pt_lt.x = rc.left_; + pt_lt.y = rc.top_; + polygon_pts.push_back(pt_lt); + cv::Point pt_rt; + pt_rt.x = rc.left_ + rc.width_; + pt_rt.y = rc.top_; + polygon_pts.push_back(pt_rt); + cv::Point pt_rb; + pt_rb.x = rc.left_ + rc.width_; + pt_rb.y = rc.top_ + rc.height_; + polygon_pts.push_back(pt_rb); + cv::Point pt_lb; + pt_lb.x = rc.left_; + pt_lb.y = rc.top_ + rc.height_; + polygon_pts.push_back(pt_lb); + + int coming_count = 0; + int leaving_count = 0; + for (size_t i = 0; i < vec_direct.size(); i++) { + SegInfo& region = vec_direct[i]; + if (region.seg_type == 1) { + std::vector vec_pt; + for (size_t j = 0; j < region.vec_pt.size(); j++) { + double dist = pointPolygonTest(polygon_pts, cv::Point2f(region.vec_pt[j].x_, region.vec_pt[j].y_), false); + if (dist > 0) { + coming_count ++; + } + } + } else if (region.seg_type == 2) { + std::vector vec_pt; + for (size_t j = 0; j < region.vec_pt.size(); j++) { + double dist = pointPolygonTest(polygon_pts, cv::Point2f(region.vec_pt[j].x_, region.vec_pt[j].y_), false); + if (dist > 0) { + leaving_count ++; + } + } } - vec_seg.push_back(info); } - return vec_seg; -} - -cv::Mat RoadSegAnalysis::parse_direct(rs_result one_result, sy_img src){ - int src_width = src.w_; - - std::vector, int>> combined; - lanes_process(one_result.reg_array, one_result.lane_count, combined); - std::vector> poly_masks, lanes, direct_masks, merge_masks; - std::vector region_classes, cats, direct_classes, merge_classes; - cv::Mat background_mask; - std::map x_sort; - bool large_resolution = false; - if (src_width > 1920) large_resolution = true; - cv::Mat seg_output = seg_post_process(large_resolution, one_result.seg_array, combined, poly_masks, region_classes, lanes, cats, x_sort, background_mask); //m_masks:mask前的结果 poly_masks后的结果 - cv::Mat direct_output = direct_post_process(large_resolution, one_result.direct_seg, direct_masks, direct_classes); - - - // std::vector> direct_masks; - // std::vector direct_classes; - // bool large_resolution = false; - // if (src_width > 1920) large_resolution = true; - // cv::Mat direct_output = direct_post_process(large_resolution, one_result.direct_seg, direct_masks, direct_classes); - - // cv::Mat image(cv::Size(640, 360), CV_8UC3, cv::Scalar(0, 0, 0)); - // cv::Mat merge_image = mask_to_rgb(image, direct_output); - // cv::imwrite("direct.jpg", merge_image); - - int direct = direct_output.at(455, 146); + int direct = 0; + if(coming_count > 0 && leaving_count > 0) { + if (coming_count > leaving_count) { + direct = 1; + } else { + direct = 2; + } + } else if(coming_count > 0 && leaving_count <= 0) { + direct = 1; + } else if(coming_count <= 0 && leaving_count > 0) { + direct = 2; + } - return direct_output; -} -// head_or_tail 0：车头 1：车尾 -int RoadSegAnalysis::check_reverse_driving(cv::Mat direct_mask, sy_rect rc, int src_width, int src_height, int head_or_tail) { - // cv::Mat image(cv::Size(640, 360), CV_8UC3, cv::Scalar(0, 0, 0)); - // cv::Mat merge_image = mask_to_rgb(image, direct_mask); - // cv::imwrite("direct_mask.jpg", merge_image); + // int center_x = (rc.left_ + rc.width_ / 2) * scale_w; + // int center_y = (rc.top_ + rc.height_ / 2) * scale_h; + // if(center_x < 0 ||center_x >= direct_mask.cols || center_y < 0 || center_y >= direct_mask.rows){ + // return -1; + // } - float scale_w = 640.0 / src_width; - float scale_h = 360.0 / src_height; - int center_x = (rc.left_ + rc.width_ / 2) * scale_w; - int center_y = (rc.top_ + rc.height_ / 2) * scale_h; - if(center_x < 0 ||center_x >= direct_mask.cols || center_y < 0 || center_y >= direct_mask.rows){ - return -1; - } // '来': 1, '去': 2, '近': 3, '远': 4 - int direct = direct_mask.at(center_x, center_y); + // int direct = direct_mask.at(center_x, center_y); if (direct == 1 && head_or_tail == 0) { // 来车道，车头，正常行驶 return 0; @@ -728,7 +259,16 @@ int RoadSegAnalysis::check_reverse_driving(cv::Mat direct_mask, sy_rect rc, int return -1; } -int RoadSegAnalysis::check_cross_line(std::vector& vec_line, sy_rect rc) { +int RoadSegAnalysis::check_cross_line(std::vector& vec_line, sy_rect src_rc) { + + float scale_w = 640.0 / src_rc.width_; + float scale_h = 360.0 / src_rc.height_; + + sy_rect rc; + rc.left_ = src_rc.left_ * scale_w; + rc.width_ = src_rc.width_ * scale_w; + rc.top_ = src_rc.top_ * scale_h; + rc.height_ = src_rc.height_ * scale_h; std::vector polygon_pts; cv::Point pt_lt; @@ -816,7 +356,123 @@ int RoadSegAnalysis::check_cross_region(std::vector& vec_reg, sy_rect r return -1; } -cv::Mat imshow_lanes(cv::Mat img, const rs_lane* lanes, int lane_count) { +// cv::Mat imshow_lanes(cv::Mat img, const rs_lane* lanes, int lane_count) { +// float scale_w = img.cols / 640.0; +// float scale_h = img.rows / 360.0; +// std::vector, int>> combined; +// lanes_process(lanes, lane_count, combined, scale_w, scale_h); + +// for (const auto& lane_info : combined) { +// const auto& xys = lane_info.first; +// int cls = lane_info.second; +// cv::Scalar color(lane_colors[cls][0],lane_colors[cls][1],lane_colors[cls][2]); +// for (size_t i = 1; i < xys.size(); ++i) { +// cv::line(img, xys[i - 1], xys[i], color, 4); +// } +// } + +// return img; +// } + +// void RoadSegAnalysis::post_direct(rs_result one_result, sy_img src){ +// int src_width = src.w_; +// int src_height = src.h_; +// int w = 640; +// int h = 360; +// float alpha = 0.75; +// cv::Mat overlayed_direct_img(cv::Size(w,h), CV_8UC3, cv::Scalar(0, 0, 0)); +// // 将车道线标签转换为彩色图像 +// for (int i = 0; i < h; ++i) { +// for (int j = 0; j < w; ++j) { +// int idx = one_result.direct_seg[(i * w + j)]; +// overlayed_direct_img.at(i, j)[0] = seg_colors[idx][0]; // R通道; +// overlayed_direct_img.at(i, j)[1] = seg_colors[idx][1]; // G通道 +// overlayed_direct_img.at(i, j)[2] = seg_colors[idx][2]; // B通道 +// } +// } +// cv::resize(overlayed_direct_img, overlayed_direct_img, cv::Size(src_width,src_height), 0, 0, cv::INTER_LINEAR); +// // 将原始图像和彩色车道线图进行混合 +// // cv::addWeighted(cvImg, alpha, overlayed_direct_img, 1 - alpha, 0, overlayed_direct_img); +// cv::Mat img_direct_lane = imshow_lanes(overlayed_direct_img, one_result.reg_array, one_result.lane_count); +// cv::imwrite("img_direct_lane.jpg", img_direct_lane); + + +// std::vector, int>> combined; +// lanes_process(one_result.reg_array, one_result.lane_count, combined); +// std::vector> poly_masks, lanes, direct_masks, merge_masks; +// std::vector region_classes, cats, direct_classes, merge_classes; +// cv::Mat background_mask; +// std::map x_sort; +// bool large_resolution = false; +// if (src_height > 1920) large_resolution = true; +// cv::Mat seg_output = seg_post_process(large_resolution, one_result.seg_array, combined, poly_masks, region_classes, lanes, cats, x_sort, background_mask); //m_masks:mask前的结果 poly_masks后的结果 +// cv::Mat direct_output = direct_post_process(large_resolution, one_result.direct_seg, direct_masks, direct_classes); +// cv::Mat merge_output = merge_direct_process(large_resolution, one_result.direct_seg, merge_masks, merge_classes); + +// cv::Mat image(cv::Size(w,h), CV_8UC3, cv::Scalar(0, 0, 0)); +// cv::Mat vis_image = mask_to_rgb(image, seg_output); +// cv::Mat direct_image = mask_to_rgb(image, direct_output); +// cv::Mat merge_image = mask_to_rgb(image, merge_output); +// cv::imwrite("vis_image.jpg", vis_image); +// cv::imwrite("direct_image.jpg", direct_image); +// cv::imwrite("merge_image.jpg", merge_image); +// cv::imwrite("background_mask.jpg", background_mask); +// } + + +cv::Mat RoadSegAnalysis::mask_to_rgb(cv::Mat img, cv::Mat mask) { + cv::Mat masks = img.clone(); + int reg_cls = 8; + for (int i = 0; i < masks.rows; i++) { + for (int j = 0; j < masks.cols; j++) { + for (int k = 1; k < reg_cls; k++) { + if (mask.at(i,j) == k) { + masks.at(i,j)[0] = seg_colors[k][0]; + masks.at(i,j)[1] = seg_colors[k][1]; + masks.at(i,j)[2] = seg_colors[k][2]; + } + } + } + } + return masks; +} + +float RoadSegAnalysis::contourArea(std::vector contour, cv::Point2f& center) { + cv::RotatedRect rect = cv::minAreaRect(contour); // 最小外接矩形 rect[0]中心点 rect[1]宽 高 rect[2]旋转角度 + center = rect.center; + return cv::contourArea(contour); +} + +void RoadSegAnalysis::lanes_process(const rs_lane* lanes, int lane_count, std::vector, int>>& combined, float scale_w, float scale_h) { + std::vector > lanes_xys; + std::vector lanes_cls; + for (int i = 0; i < lane_count; i++) { + std::vector xys; + for (int j = 0; j < lanes[i].num_points; j++) { + int x = static_cast(lanes[i].points[j].x_ * scale_w); + int y = static_cast(lanes[i].points[j].y_ * scale_h); + if (x > 0 && y > 0) { + xys.emplace_back(x, y); + } + } + if (!xys.empty()) { + lanes_xys.push_back(xys); + lanes_cls.push_back(lanes[i].cls); + } + } + + for (size_t i = 0; i < lanes_xys.size(); ++i) { + combined.push_back(std::make_pair(lanes_xys[i], lanes_cls[i])); + } + if (!combined.empty()) { + //按车道线起点坐标排序,相应的类别顺序也会变化以保证标签对齐 + std::sort(combined.begin(), combined.end(), [](const std::pair, int>& a, const std::pair, int>& b) { + return a.first[0].x < b.first[0].x; + }); + } +} + +cv::Mat RoadSegAnalysis::imshow_lanes(cv::Mat img, const rs_lane* lanes, int lane_count) { float scale_w = img.cols / 640.0; float scale_h = img.rows / 360.0; std::vector, int>> combined; @@ -834,47 +490,145 @@ cv::Mat imshow_lanes(cv::Mat img, const rs_lane* lanes, int lane_count) { return img; } -void RoadSegAnalysis::post_direct(rs_result one_result, sy_img src){ - int src_width = src.w_; - int src_height = src.h_; - int w = 640; - int h = 360; - float alpha = 0.75; - cv::Mat overlayed_direct_img(cv::Size(w,h), CV_8UC3, cv::Scalar(0, 0, 0)); - // 将车道线标签转换为彩色图像 - for (int i = 0; i < h; ++i) { - for (int j = 0; j < w; ++j) { - int idx = one_result.direct_seg[(i * w + j)]; - overlayed_direct_img.at(i, j)[0] = seg_colors[idx][0]; // R通道; - overlayed_direct_img.at(i, j)[1] = seg_colors[idx][1]; // G通道 - overlayed_direct_img.at(i, j)[2] = seg_colors[idx][2]; // B通道 +int RoadSegAnalysis::Mask2LanePoints(const cv::Mat& pred, std::vector>&lanes, std::vector& cats) { + std::vector labels = {8, 9, 10, 11}; + for(auto cat: labels) { + cv::Mat b_masks, measure_labels, stats, centroids; + cv::compare(pred, cat, b_masks, cv::CMP_EQ); //将pred元素逐个和cat比较，相同255，不同0 + // cv::threshold(pred, b_masks, 126, 255, cv::THRESH_OTSU); //生成二值图 + // 连通域计算 b_masks:闭操作后的二值图像 measure_labels:和原图一样大的标记图 centroids:nccomps×2的矩阵,表示每个连通域的质心(x, y) + // stats:nccomps×5的矩阵 表示每个连通区域的外接矩形和面积(x, y, w, h, area)，索引0是背景信息 + int nccomps = cv::connectedComponentsWithStats (b_masks, measure_labels, stats, centroids, 4); //8 + for(int cv_measure_id = 1; cv_measure_id < nccomps; cv_measure_id++ ) { //跳过背景信息0 + cv::Mat cv_measure_mask; + cv::compare(measure_labels, cv_measure_id, cv_measure_mask, cv::CMP_EQ); + std::vector< std::vector< cv::Point> > contours; + cv::findContours(cv_measure_mask,contours,cv::noArray(),cv::RETR_TREE,cv::CHAIN_APPROX_SIMPLE); + std::vector contours_poly; + for (int j = 0; j < contours.size();j++) { + float area = cv::contourArea(contours[j]); if (area < 60) continue; //30 + cv::approxPolyDP(cv::Mat(contours[j]), contours_poly, 6, true); //减小epsilon提高拟合精度(需要调试，epsilon过小会使多边形不够简化，单条线变多条) + if (contours_poly.size() == 1) continue; + lanes.push_back(contours_poly); + cats.push_back(cat); + } + } + } + return 0; +} + + +cv::Mat RoadSegAnalysis::seg_post_process(bool large_resolution, unsigned char *seg_array, std::vector, int>> combined, std::vector> &poly_masks, std::vector ®ion_classes, std::vector> &lanes, std::vector &cats, std::map &x_sort) { + std::vector pred_cls; + int h = 360, w = 640; + cv::Mat lanes_masks = cv::Mat_(h,w); //正常分割结果 + cv::Mat mask_rmlane = cv::Mat_(h,w); //车道线区域置为背景 + cv::Mat solve_masks = cv::Mat_(h,w); //计算主行驶区域用(虚线及减速标线归入行车道) + int step_size = h*w; + int seg_min_region_area = 512; //1024 + + for (int i = 0; i < h; ++i) { + for (int j = 0; j < w; ++j) { + int max_cls = seg_array[(i * w + j)]; + lanes_masks.ptr(i)[j] = max_cls; + pred_cls.push_back(max_cls); + mask_rmlane.ptr(i)[j] = max_cls; + solve_masks.ptr(i)[j] = max_cls; + } + } + + for (const auto& lane_info : combined) { + const auto& xys = lane_info.first; + int cls = lane_info.second; + if (cls == 1) cls = 8; /*黄实线*/ if (cls == 2) cls = 9; /*白实线*/ + if (cls == 3) cls = 10; /*虚线*/ if (cls == 4) cls = 11; /*黄虚线*/ + for (size_t i = 1; i < xys.size(); ++i) { + cv::line(lanes_masks, xys[i - 1], xys[i], cls, 4); //绘制车道线用于求连通域 + if (cls == 10 || cls == 11) cls = 1; //求主行驶区域时将虚线归入行车道 + cv::line(solve_masks, xys[i - 1], xys[i], cls, 4); + } } - cv::resize(overlayed_direct_img, overlayed_direct_img, cv::Size(src_width,src_height), 0, 0, cv::INTER_LINEAR); - // 将原始图像和彩色车道线图进行混合 - // cv::addWeighted(cvImg, alpha, overlayed_direct_img, 1 - alpha, 0, overlayed_direct_img); - cv::Mat img_direct_lane = imshow_lanes(overlayed_direct_img, one_result.reg_array, one_result.lane_count); - cv::imwrite("img_direct_lane.jpg", img_direct_lane); + /* 求背景区域--mask车道区域，场景变化用 + cv::compare(lanes_masks, 0, background_mask, cv::CMP_EQ); //将lanes_masks元素逐个和0比较，相同255，不同0*/ +#if 1 + //--mask远处区域------------------------------------------------ + cv::Mat mask_black = mask_rmlane.clone(); + if (large_resolution) mask_black(cv::Rect(0, 0, w, int(h * 0.14))) = 0; + else mask_black(cv::Rect(0, 0, w, int(h * 0.22))) = 0; + mask_black(cv::Rect(0, int(h * 0.95), w, int(h * 0.05))) = 0; + mask_black(cv::Rect(0, 0, int(w * 0.02), h)) = 0; + mask_black(cv::Rect(int(w * 0.95), 0, int(w * 0.05), h)) = 0; + mask_rmlane = mask_black; + //------------------------------------------------------------------------- +#endif + //2.去重获取预测到的类别 + std::vector labels(pred_cls); + std::sort(labels.begin(),labels.end()); + labels.erase(std::unique(labels.begin(),labels.end()),labels.end()); + + //4.求车道线区域 + int flag = Mask2LanePoints(lanes_masks, lanes, cats); ///////////////////////// 车道线如何与mask结合 + + //5.求道路区域 + int count = 0; + for(auto cat: labels) { + // std::cout << cat << std::endl; + cv::Mat b_masks, measure_labels, stats, centroids; + cv::Mat n_masks = cv::Mat_(h,w); + n_masks = cv::Scalar(255); + cv::compare(mask_rmlane, cat, b_masks, cv::CMP_EQ); //将mask_rmlane元素逐个和cat比较，相同255，不同0 + //连通域计算 + int nccomps = cv::connectedComponentsWithStats (b_masks, measure_labels, stats, centroids, 8); + for(int i = 1; i < nccomps; i++ ) { //跳过背景信息0 + //移除过小的区域，并将对应位置置为0 + if (stats.at(i, cv::CC_STAT_AREA) < seg_min_region_area) { + cv::Mat comparison_result; + cv::compare(measure_labels, cv::Scalar(i), comparison_result, cv::CMP_EQ); //相等为255不等为0 + n_masks.setTo(0, comparison_result); // 将comparison_result中非零区域置为0 + + cv::bitwise_and(mask_rmlane,n_masks,mask_rmlane); + continue; + } - std::vector, int>> combined; - lanes_process(one_result.reg_array, one_result.lane_count, combined); - std::vector> poly_masks, lanes, direct_masks, merge_masks; - std::vector region_classes, cats, direct_classes, merge_classes; - cv::Mat background_mask; - std::map x_sort; - bool large_resolution = false; - if (src_height > 1920) large_resolution = true; - cv::Mat seg_output = seg_post_process(large_resolution, one_result.seg_array, combined, poly_masks, region_classes, lanes, cats, x_sort, background_mask); //m_masks:mask前的结果 poly_masks后的结果 - cv::Mat direct_output = direct_post_process(large_resolution, one_result.direct_seg, direct_masks, direct_classes); - cv::Mat merge_output = merge_direct_process(large_resolution, one_result.direct_seg, merge_masks, merge_classes); - - cv::Mat image(cv::Size(w,h), CV_8UC3, cv::Scalar(0, 0, 0)); - cv::Mat vis_image = mask_to_rgb(image, seg_output); - cv::Mat direct_image = mask_to_rgb(image, direct_output); - cv::Mat merge_image = mask_to_rgb(image, merge_output); - cv::imwrite("vis_image.jpg", vis_image); - cv::imwrite("direct_image.jpg", direct_image); - cv::imwrite("merge_image.jpg", merge_image); - cv::imwrite("background_mask.jpg", background_mask); + double centr_x = centroids.at(i, 0); + double centr_y = centroids.at(i, 1); + + int region_class = cat; + // printf("region_class: %d\n", region_class); + cv::Mat region_mask; + cv::bitwise_and(measure_labels,n_masks,measure_labels);///// + cv::compare(measure_labels, i, region_mask, cv::CMP_EQ); + + std::vector< std::vector< cv::Point> > contours; + cv::findContours(region_mask,contours,cv::noArray(),cv::RETR_TREE,cv::CHAIN_APPROX_SIMPLE); + std::vector contours_poly; + for (int j = 0; j < contours.size();j++) { + cv::approxPolyDP(cv::Mat(contours[j]), contours_poly, 10, true); + if (contours_poly.size() <= 2) continue; + poly_masks.push_back(contours_poly); + region_classes.push_back(cat); + if (x_sort.count(centr_x)) centr_x += 0.0001; + x_sort.insert(std::make_pair(centr_x, count)); + ++ count; + // for (auto iter: contours_poly) { + // std::cout << "contour " << iter << " " << cat << std::endl; + // } + + } + } + } + +#if 0 + //6.draw lanes + for (int i = 0; i < lanes.size(); ++i) { + int thickness = 4; + for (int j = 0; j < lanes[i].size()-1; ++j) { + cv::line(mask_rmlane, lanes[i][j], lanes[i][j+1], {seg_colors[cats[i]][0],seg_colors[cats[i]][1],seg_colors[cats[i]][2]}, thickness); + // std::cout << lanes[i][j] << " " << lanes[i][j+1] << " " << cats[i] << std::endl; + } + } +#endif + return mask_rmlane; } \ No newline at end of file diff --git a/src/ai_engine_module/RoadSegAnalysis.h b/src/ai_engine_module/RoadSegAnalysis.h index 2b40563..20dda80 100644 --- a/src/ai_engine_module/RoadSegAnalysis.h +++ b/src/ai_engine_module/RoadSegAnalysis.h @@ -9,8 +9,7 @@ using namespace std; struct RoadInfo{ std::vector vec_line; std::vector vec_road; - - cv::Mat direct_mask; + std::vector vec_direct; }; class RoadSegAnalysis @@ -23,7 +22,7 @@ public: std::vector detect(vector vec_img); - int check_reverse_driving(cv::Mat direct_mask, sy_rect rc, int src_width, int src_height, int head_or_tail); + int check_reverse_driving(std::vector& vec_direct, sy_rect rc, int src_width, int src_height, int head_or_tail); int check_cross_line(std::vector& vec_line, sy_rect rc); @@ -32,11 +31,20 @@ public: private: int release(); - std::vector parse_seg(rs_result one_result, sy_img src); + RoadInfo parse_road(rs_result one_result, sy_img src); + + std::vector parse_direct(rs_result one_result, sy_img src); + + // void post_direct(rs_result one_result, sy_img src); + - cv::Mat parse_direct(rs_result one_result, sy_img src); + cv::Mat mask_to_rgb(cv::Mat img, cv::Mat mask); + float contourArea(std::vector contour, cv::Point2f& center); + void lanes_process(const rs_lane* lanes, int lane_count, std::vector, int>>& combined, float scale_w = 1.0, float scale_h = 1.0); + cv::Mat imshow_lanes(cv::Mat img, const rs_lane* lanes, int lane_count); + int Mask2LanePoints(const cv::Mat& pred, std::vector>&lanes, std::vector& cats); + cv::Mat seg_post_process(bool large_resolution, unsigned char *seg_array, std::vector, int>> combined, std::vector> &poly_masks, std::vector ®ion_classes, std::vector> &lanes, std::vector &cats, std::map &x_sort); - void post_direct(rs_result one_result, sy_img src); private: void* m_handle{nullptr}; aclrtContext ctx{nullptr}; diff --git a/src/village_inc.h b/src/village_inc.h index 944f987..cce03b7 100644 --- a/src/village_inc.h +++ b/src/village_inc.h @@ -36,10 +36,11 @@ typedef struct VehicleInfo { manned_result manned_res; int type; - + int vpt_type; int rainshed; //是否安装雨棚， 0 有雨棚 1 无雨棚 - int manned; + int truck_manned; + int motor_manned; int human_upper_color; int human_lower_color; -- libgit2 0.21.4