cv2.imdecode() 函数从指定的内存缓存中读取数据，并把数据转换(解码)成图像格式;主要用于从网络传输数据中恢复出图像.cv2.imencode() 函数是将图片格式转换(编码)成流数据，赋值到内存缓存中;主要用于图像数据格式的压缩，方便网络传输.1. cv2.imdecode()读取网络url图像数据，并转换成图片格式：#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import numpy as np import cv2 import requests url = 'https://www.aiuai.cn/uploads/2202/beeb259c1c4056ce.jpeg' resp = requests.get(url, timeout=3) image = np.asarray(bytearray(resp.content), dtype="uint8") image = cv2.imdecode(image, cv2.IMREAD_COLOR) cv2.imshow('url_image',image) cv2.waitKey()2. cv2.imencode()将图片进行编码，并保存到本地:#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import numpy as np import cv2 img = cv2.imread('test.jpg') img_encode = cv2.imencode('.jpg', img)[1] #.jpg 编码格式 # img_encode = cv2.imencode('.png', img) #.png 编码格式 img_array = np.array(img_encode) img_str = img_array.tostring() # 缓存数据保存到本地 with open('img_encode.txt', 'w') as f: f.write(img_str) f.flush3. cv2.imencode() + cv2.imdecode()图片编码保存到本地，读取本地文件解码恢复成图片格式：#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import numpy as np import cv2 #imencode img = cv2.imread('test.jpg') img_encode = cv2.imencode('.jpg', img)[1] #.jpg 编码格式 # img_encode = cv2.imencode('.png', img) #.png 编码格式 img_array = np.array(img_encode) img_str = img_array.tostring() # 缓存数据保存到本地 with open('img_encode.txt', 'w') as f: f.write(img_str) f.flush #imdecode with open('img_encode.txt', 'r') as f: img_str = f.read() img_array = np.fromstring(img_str, np.uint8) #img_array = np.asarray(bytearray(img_str), dtype="uint8") img_decode = cv2.imdecode(img_array, cv2.IMREAD_COLOR) cv2.imshow("img_decode", img_decode) cv2.waitKey()4. 性能提升OpenCV 是基于libjpeg 库进行图片编解码，imdecode 速度可能会慢.这里采用 turbojpeg 库进行速度提升.4.1. cv2.imdecode()#cv2.imdecode img_cv2 = base64.b64decode(data) img_array = np.frombuffer(img_cv2, np.uint8) img_decode = cv2.imdecode(img_array, cv2.IMREAD_COLOR) #turbojpeg from turbojpeg import TurboJPEG,TJPF_GRAY,TJSAMP_GRAY,TJFLAG_PROGRESSIVE,TJPF_BGR turboJpeg_lib_path = '/path/to/libTurboJpeg.so' jpeg = TurboJPEG(turboJpeg_lib_path) img = base64.b64decode(data) img_decode = jpeg.decode(img, pixel_format=TJPF_BGR)4.2. cv2.imwrite()#cv2.imwrite cv2.imwrite('img.jpg',img) #turbojpeg from turbojpeg import TurboJPEG,TJPF_GRAY,TJSAMP_GRAY,TJFLAG_PROGRESSIVE,TJPF_BGR turboJpeg_lib_path = '/path/to/libTurboJpeg.so' jpeg = TurboJPEG(turboJpeg_lib_path) # with open('img.jpg', 'wb') as fp: fp.write( jpeg.encode(image,quality=100))材料[1] - OpenCV-Python cv2.imdecode()和cv2.imencode() 图片解码和编码 - 2018.01.25[2] - 关于opencv imdecode速度慢的问题 - 2021.09.18
#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import cv2 img = cv2.imread("elon.jpg") ## Image to Gray Image gray_image = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY) ## Gray Image to Inverted Gray Image inverted_gray_image = 255-gray_image ## Blurring The Inverted Gray Image blurred_inverted_gray_image = cv2.GaussianBlur(inverted_gray_image, (19,19),0) ## Inverting the blurred image inverted_blurred_image = 255-blurred_inverted_gray_image ### Preparing Photo sketching sketck = cv2.divide(gray_image, inverted_blurred_image,scale= 256.0) cv2.imshow("Original Image",img) cv2.imshow("Pencil Sketch", sketck) cv2.waitKey(0)如图：
Ubuntu20.04 编译 Caffe 时与 OpenCV4.X 的问题记录与解决.Caffe - Ubuntu 安装与测试实例及问题解决 - AIUAIUbuntu - OpenCV 源码安装与测试 - AIUAI参考：How to Install OpenCV on Ubuntu 20.04 - 2020.07.05参考：Install OpenCV 4.4.0 and Caffe on Ubuntu 20.04 for Python 3Ubuntu20.04 仓库安装 OpenCV:sudo apt install libopencv-dev python3-opencv查看版本：pkg-config --modversion opencv4 # 4.2.0问题1 - src/caffe/data_transformer.cpp:2:33: fatal error: opencv2/core/core.hpp原因：找不到头文件.解决方案：将 OpenCV 的头文件加入：原： INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial/ 新： INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial/ /usr/include/opencv4/问题2 - error: ‘CV_INTER_CUBIC’ was not declared in this scope问题：src/caffe/data_transformer.cpp:781:84: error: ‘CV_INTER_CUBIC’ was not declared in this scope cv::resize(crop_bufferM, cv_cropped_img, cv::Size(crop_size, crop_size), 0, 0, CV_INTER_CUBIC); ^~~~~~~~~~~~~~ src/caffe/data_transformer.cpp:781:84: note: suggested alternative: ‘SA_INTERRUPT’ cv::resize(crop_bufferM, cv_cropped_img, cv::Size(crop_size, crop_size), 0, 0, CV_INTER_CUBIC); ^~~~~~~~~~~~~~ SA_INTERRUPT src/caffe/data_transformer.cpp: In member function ‘void caffe::DataTransformer<Dtype>::Transform(const caffe::Datum&, const caffe::Datum&, caffe::Blob<Dtype>*, caffe::Blob<Dtype>*)’: src/caffe/data_transformer.cpp:954:51: error: ‘CV_INTER_NN’ was not declared in this scope cv::resize(M, M, cv::Size(width, height), 0, 0, CV_INTER_NN); ^~~~~~~~~~~ src/caffe/data_transformer.cpp:954:51: note: suggested alternative: ‘CV_INLINE’ cv::resize(M, M, cv::Size(width, height), 0, 0, CV_INTER_NN); ^~~~~~~~~~~ CV_INLINE make: *** [Makefile:592: .build_release/src/caffe/data_transformer.o] Error 1解决方案：在 src/caffe/data_transformer.cpp 中添加：include <opencv2/imgproc/imgproc_c.h>：#ifdef USE_OPENCV #include <opencv2/core/core.hpp> #include <opencv2/core/core.hpp> #include <opencv2/core/mat.hpp> #include <opencv2/highgui/highgui.hpp> #include <opencv2/imgproc/imgproc.hpp> #include <opencv2/imgproc/imgproc_c.h> // new add #include <opencv2/opencv.hpp> #endif // USE_OPENCV问题3 - error: ‘CV_LOAD_IMAGE_COLOR’ was not declared in this scope问题：src/caffe/layers/window_data_layer.cpp: In member function ‘virtual void caffe::WindowDataLayer<Dtype>::load_batch(caffe::Batch<Dtype>*)’: src/caffe/layers/window_data_layer.cpp:293:42: error: ‘CV_LOAD_IMAGE_COLOR’ was not declared in this scope cv_img = cv::imread(image.first, CV_LOAD_IMAGE_COLOR); ^~~~~~~~~~~~~~~~~~~ src/caffe/layers/window_data_layer.cpp:293:42: note: suggested alternative: ‘CV_IS_IMAGE_HDR’ cv_img = cv::imread(image.first, CV_LOAD_IMAGE_COLOR); ^~~~~~~~~~~~~~~~~~~ CV_IS_IMAGE_HDR make: *** [Makefile:592: .build_release/src/caffe/layers/window_data_layer.o] Error 1原因：OpenCV 版本问题，OpenCV4 变量名发生变动.解决方案：在 include/caffe/common.hpp 中新增如下代码(大概 70 行前后)：// Supporting OpenCV4 #if (CV_MAJOR_VERSION == 4) #define CV_LOAD_IMAGE_COLOR cv::IMREAD_COLOR #define CV_LOAD_IMAGE_GRAYSCALE cv::IMREAD_GRAYSCALE #endif //以上为新增代码(以下为原有代码，用于定位新增代码位置) // See PR #1236 namespace cv { class Mat; }另一种解决方案：对报错文件进行如下修改：CV_LOAD_IMAGE_COLOR 修改为 cv::IMREAD_COLOR CV_LOAD_IMAGE_GRAYSCALE 修改为 cv::IMREAD_GRAYSCALE
4原文：Mask R-CNN with OpenCV - 2018.11.19作者：Adrian Rosebrock这篇博文主要是介绍如何基于 OpenCV 使用 Mask R-CNN.Mask R-CNN 可以自动对图片中每个目标的进行分割，并构建像素级的 masks.基于OpenCV DNN的 MaskRCNN 目标检测与实例分割 - AIUAI在博文 YOLOV3 - 基于 OpenCV 的 YOLO 目标检测 - AIUAI 中介绍了基于 OpenCV 的 YOLO 目标检测算法. 目标检测器，如 YOLO，Faster R-CNNs 和 SSDs，输出的是图片中每个目标物体的边界框的 (x, y) 坐标集合. 但是边界框缺乏一定的像素级信息，如 (1)哪些像素属于前景目标；(2) 哪些像素属于背景.博文主要包括：[1] - Mask R-CNN 概览[2] - Mask R-CNN 图片[3] - Mask R-CNN 视频流1. Mask R-CNN 概览1.1. 实例分割 vs. 语义分割如下图，很好的说明了图像分类、目标检测、语义分割和实例分割之间的差异.[1] - 图像分类，旨在预测能够刻画输入图像内容的标签集.[2] - 目标检测，基于图像分类，但是同时会定位图片中目标的位置，则图像的刻画即包括两部分: (1) 每个目标的边界框(x,y) 坐标; (2) 对应每个边界框的相关类别标签.[3] - 语义分割，预测输入图像的每个像素的类别标签(包括背景的类别标签.) 注：上图(c)中 cube 目标均是相同的颜色. 虽然语义分割算法能够对每个目标的像素进行分类，但是不能区分相同类别的两个目标.[4] - 实例分割，即使图像中存在相同类别标签的目标，也能够预测图片中每个目标的像素级 mask. 如上图(d), 每个 cube 都是不同的颜色. 即：实例分割不仅能够定位每个独立的 cube，还能够预测其边界位置.Mask R-CNN 是一种实例分割算法.1.2. Mask R-CNN 简介Mask R-CNN 算法是 He 等在论文 Mask R-CNN(2017) 中提出的.Mask R-CNN 是基于 Girshick 等人在 R-CNN (2013), Fast R-CNN (2015) 和 Faster R-CNN (2015) 中的目标检测工作的.为了更好的理解 Mask R-CNN 算法，这里简单回顾下 R-CNNs:[1] - R-CNN主要包括四步处理：(1)-Step1: 输入一张图片到网络；(2)-Step2: 提取区域候选框(region proposals)，例如采用 Selective Search 算法得到的可能包含目标物体的图片区域.(3)-Step3: 采用迁移学习，即特征提取部分，采用预训练的CNN计算每个候选框(ROI)的特征.(4)-Step4: 采用 SVM 对每个候选框的提取特征进行分类.R-CNN 有效的原因在于 CNN 提取的鲁棒、判别性强的特征. 但，R-CNN 的问题在于其速度相当的慢. 此外，其并不是真正通过深度神经网络来学习定位目标，实际上只是构建了更高级的 HOG + Linear SVM detector 方法.[2] - Fast R-CNN类似于 R-CNN，Fast R-CNN 也采用了 Selective Search 算法来获得区域候选框；不过，其创新点在于 Region of Interest (ROI) Pooling 模块.ROI Pooling 通过对 CNN 的输出的 feature map 提取固定尺寸的窗口(fixed-size window)，并采用窗口内的特征得到最终的类别标签和边界框. Fast R-CNN 最重要的提升是，有效的实现了网络的 end-to-end 训练，其过程主要为：(1)-给定输入图片及对应的 ground-truth 边界框;(2)-提取特征图(feature map);(3)-采用 ROI Pooling 处理，得到 ROI 特征向量;(4)-最后，采用两个 FC 层，同时得到每个候选框的类别标签预测和边界框位置.虽然 Fast R-CNN 是 end-to-end 训练的，但是由于依赖于 Selective Search 算法，其在预测推断时，速度明显受影响.[3] - Faster R-CNN为了进一步提升 R-CNN 结构的速度，Faster R-CNN 直接将区域候选框选取(region propoasal) 整合进网络结构中.**Faster R-CNN 提出 Region Proposal Network (RPN)，以实现将区域候选框选取(region propoasal) 直接整合进网络中，取代 Selective Search 算法.Faster R-CNN 结构能够取得 7-10 FPS 的速度，将基于深度学习的实时目标检测提升了一大步.[4] - Mask R-CNNMask R-CNN 是基于 Faster R-CNN 结构构建的，主要有两个创新：(1)-将 ROI Pooling 模块替换为 ROI Align 模块；(2)-在ROI Align 模块的输出端新增一个分支. 该分支的输入为 ROI Align 的输出，该分支的输出送入到两个 Conv 层. Conv 层的输出即为 mask.如图：正如在 Faster R-CNN/Mask R-CNN 结构中，采用RPN 网络来生成图片中可能包含物体的区域候选框. 每个候选区域根据其"目标分数(objectness score)" 进行排名(如，表征了给定区域可能包含目标的可能性), 然后保留 top N 个最可能包含目标的区域.在 Faster R-CNN 论文中，Girshick 等人设置 N=2000. 但实际上，可以采用较小的 N 值，如 N=(10, 100, 200, 300)，仍能得到比较好的结果.在 Mask R-CNN 论文中，He 等设置 N=300，这里也采用该设置.所选择的 300 个 ROIs 被送入到三个并行的网络分支：(1)类别标签预测(2)边界框预测(3)Mask 预测在 Mask R-CNN 预测阶段，所选择的 300 个 ROIs 先送入 NMS( non-maxima suppression) 处理，并只保留 top 100 的检测框，最终得到 100xLx15x15 的 4D Tensor. 其中，L 是数据集中类别标签的总数，15x15 是每个 mask 的尺寸.以 COCO dataset 为例，L=90 个目标物体类别标签，mask 预测分支得到的结果为 100x90x15x15.Mask R-CNN 预测 mask 的过程如图：更多细节参考：[1] - Mask R-CNN[2] - Deep Learning for Computer Vision with Python2. Mask R-CNN 图片实例分割具体实现如下 - mask_rcnn.py：#!/usr/bin/python3 #--*-- coding:utf-8 --*-- import numpy as np import argparse import random import time import cv2 import os #参数配置 ap = argparse.ArgumentParser() ap.add_argument("-i", "--image", required=True, help="path to input image") ap.add_argument("-m", "--mask-rcnn", required=True, help="base path to mask-rcnn directory") ap.add_argument("-v", "--visualize", type=int, default=0, help="whether or not we are going to visualize each instance") ap.add_argument("-c", "--confidence", type=float, default=0.5, help="minimum probability to filter weak detections") ap.add_argument("-t", "--threshold", type=float, default=0.3, help="minimum threshold for pixel-wise mask segmentation") args = vars(ap.parse_args()) #加载 COCO 类别标签 labelsPath = os.path.sep.join([args["mask_rcnn"], "object_detection_classes_coco.txt"]) LABELS = open(labelsPath).read().strip().split("\n") #加载用于可视化给定实例分割的颜色集合 colorsPath = os.path.sep.join([args["mask_rcnn"], "colors.txt"]) COLORS = open(colorsPath).read().strip().split("\n") COLORS = [np.array(c.split(",")).astype("int") for c in COLORS] COLORS = np.array(COLORS, dtype="uint8") #Mask R-CNN 权重路径及模型配置文件 weightsPath = os.path.sep.join([args["mask_rcnn"], "frozen_inference_graph.pb"]) configPath = os.path.sep.join([args["mask_rcnn"], "mask_rcnn_inception_v2_coco_2018_01_28.pbtxt"]) #加载预训练的 Mask R-CNN 模型(90 classes) print("[INFO] loading Mask R-CNN from disk...") net = cv2.dnn.readNetFromTensorflow(weightsPath, configPath) #读取图片 image = cv2.imread(args["image"]) (H, W) = image.shape[:2] #构建输入图片 blob blob = cv2.dnn.blobFromImage(image, swapRB=True, crop=False) net.setInput(blob) start = time.time() #forward 计算，输出图片中目标的边界框坐标以及每个目标的像素级分割 (boxes, masks) = net.forward(["detection_out_final", "detection_masks"]) end = time.time() # Mask R-CNN 的时间统计 print("[INFO] Mask R-CNN took {:.6f} seconds".format(end - start)) print("[INFO] boxes shape: {}".format(boxes.shape)) print("[INFO] masks shape: {}".format(masks.shape)) # loop over the number of detected objects for i in range(0, boxes.shape[2]): #检测的 class ID 及对应的置信度(概率) classID = int(boxes[0, 0, i, 1]) confidence = boxes[0, 0, i, 2] #过滤低置信度预测结果 if confidence > args["confidence"]: # 用于可视化 clone = image.copy() #将边界框坐标缩放回相对于图片的尺寸，然后计算边界框的width和height box = boxes[0, 0, i, 3:7] * np.array([W, H, W, H]) (startX, startY, endX, endY) = box.astype("int") boxW = endX - startX boxH = endY - startY #提取目标的像素级分割 mask = masks[i, classID] #resize mask以保持与边界框的维度一致 mask = cv2.resize(mask, (boxW, boxH), interpolation=cv2.INTER_NEAREST) #根据设定阈值，得到二值化mask. mask = (mask > args["threshold"]) #提取图片的 ROI roi = clone[startY:endY, startX:endX] #可视化 if args["visualize"] > 0: #将二值mask转换为:0和255 visMask = (mask * 255).astype("uint8") instance = cv2.bitwise_and(roi, roi, mask=visMask) #可视化提取的 ROI、mask 以及对应的分割实例 cv2.imshow("ROI", roi) cv2.imshow("Mask", visMask) cv2.imshow("Segmented", instance) #只提取 ROI 的 masked 区域 roi = roi[mask] #随机选择一种颜色，用于可视化特定的实例分割 color = random.choice(COLORS) #通过融合选择的颜色和 ROI 进行融合，创建透明覆盖图 blended = ((0.4 * color) + (0.6 * roi)).astype("uint8") #替换原始图片的融合 ROI 区域 clone[startY:endY, startX:endX][mask] = blended #画出图片中实例的边界框 color = [int(c) for c in color] cv2.rectangle(clone, (startX, startY), (endX, endY), color, 2) #画出预测的类别标签以及对应的实例概率 text = "{}: {:.4f}".format(LABELS[classID], confidence) cv2.putText(clone, text, (startX, startY - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2) #show cv2.imshow("Output", clone) cv2.waitKey(0)运行，如：python mask_rcnn.py --mask-rcnn mask-rcnn-coco --image images/example_01.jpg python mask_rcnn.py --mask-rcnn mask-rcnn-coco --image images/example_03.jpg --visualize 1输出如：中间输入如：3. Mask R-CNN 视频流实例分割具体实现如下 - mask_rcnn_video.py:import numpy as np import argparse import imutils import time import cv2 import os # ap = argparse.ArgumentParser() ap.add_argument("-i", "--input", required=True, help="path to input video file") ap.add_argument("-o", "--output", required=True, help="path to output video file") ap.add_argument("-m", "--mask-rcnn", required=True, help="base path to mask-rcnn directory") ap.add_argument("-c", "--confidence", type=float, default=0.5, help="minimum probability to filter weak detections") ap.add_argument("-t", "--threshold", type=float, default=0.3, help="minimum threshold for pixel-wise mask segmentation") args = vars(ap.parse_args()) # labelsPath = os.path.sep.join([args["mask_rcnn"], "object_detection_classes_coco.txt"]) LABELS = open(labelsPath).read().strip().split("\n") # np.random.seed(42) COLORS = np.random.randint(0, 255, size=(len(LABELS), 3), dtype="uint8") # weightsPath = os.path.sep.join([args["mask_rcnn"], "frozen_inference_graph.pb"]) configPath = os.path.sep.join([args["mask_rcnn"], "mask_rcnn_inception_v2_coco_2018_01_28.pbtxt"]) # print("[INFO] loading Mask R-CNN from disk...") net = cv2.dnn.readNetFromTensorflow(weightsPath, configPath) #读取视频流 vs = cv2.VideoCapture(args["input"]) writer = None #计算视频流总帧数 try: prop = cv2.cv.CV_CAP_PROP_FRAME_COUNT if imutils.is_cv2() \ else cv2.CAP_PROP_FRAME_COUNT total = int(vs.get(prop)) print("[INFO] {} total frames in video".format(total)) except: print("[INFO] could not determine # of frames in video") total = -1 # loop over frames from the video file stream while True: # read the next frame from the file (grabbed, frame) = vs.read() if not grabbed: break # blob = cv2.dnn.blobFromImage(frame, swapRB=True, crop=False) net.setInput(blob) start = time.time() (boxes, masks) = net.forward(["detection_out_final", "detection_masks"]) end = time.time() # loop over the number of detected objects for i in range(0, boxes.shape[2]): classID = int(boxes[0, 0, i, 1]) confidence = boxes[0, 0, i, 2] if confidence > args["confidence"]: (H, W) = frame.shape[:2] box = boxes[0, 0, i, 3:7] * np.array([W, H, W, H]) (startX, startY, endX, endY) = box.astype("int") boxW = endX - startX boxH = endY - startY # mask = masks[i, classID] mask = cv2.resize(mask, (boxW, boxH), interpolation=cv2.INTER_NEAREST) mask = (mask > args["threshold"]) # roi = frame[startY:endY, startX:endX][mask] # color = COLORS[classID] blended = ((0.4 * color) + (0.6 * roi)).astype("uint8") # frame[startY:endY, startX:endX][mask] = blended # color = [int(c) for c in color] cv2.rectangle(frame, (startX, startY), (endX, endY), color, 2) # text = "{}: {:.4f}".format(LABELS[classID], confidence) cv2.putText(frame, text, (startX, startY - 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2) # check if the video writer is None if writer is None: # initialize our video writer fourcc = cv2.VideoWriter_fourcc(*"MJPG") writer = cv2.VideoWriter(args["output"], fourcc, 30, (frame.shape[1], frame.shape[0]), True) # some information on processing single frame if total > 0: elap = (end - start) print("[INFO] single frame took {:.4f} seconds".format(elap)) print("[INFO] estimated total time to finish: {:.4f}".format( elap * total)) # write the output frame to disk writer.write(frame) # release the file pointers print("[INFO] cleaning up...") writer.release() vs.release()运行，如：python mask_rcnn_video.py --input videos/cats_and_dogs.mp4 \ --output output/cats_and_dogs_output.avi \ --mask-rcnn mask-rcnn-coco
在前面已经测试过 YOLOV3 和 SSD 基于 OpenCV DNN 的目标检测实现，这里再简单实现下 Faster RCNN 基于 DNN 的实现.YOLOV3 基于OpenCV DNN 的目标检测实现 - AIUAITensorFlow 目标检测模型转换为 OpenCV DNN 可调用格式 - AIUAI1. Faster RCNN 模型下载直接从 OpenCV DNN 提供的模型 weights 文件和 config 文件链接下载：ModelVersion Faster-RCNN Inception v22018_01_28weightsconfigFaster-RCNN ResNet-502018_01_28weightsconfig或者，根据 TensorFlow 目标检测模型转换为 OpenCV DNN 可调用格式 - AIUAI 中的说明，自己进行模型转化. 如果是基于 TensorFlow 对定制数据集训练的模型，则采用这种方法.这里以 faster_rcnn_resnet50_coco_2018_01_28 模型为例，手工得到 graph.pbtxt 文件，进行测试.2. Faster RCNN DNN 实现之一#!/usr/bin/python #!--*-- coding:utf-8 --*-- import cv2 import matplotlib.pyplot as plt pb_file = '/path/to/faster_rcnn_resnet50_coco_2018_01_28/frozen_inference_graph.pb' pbtxt_file = '/path/to/faster_rcnn_resnet50_coco_2018_01_28/graph.pbtxt' net = cv2.dnn.readNetFromTensorflow(pb_file, pbtxt_file) score_threshold = 0.3 img_file = "test.jpg" img_cv2 = cv2.imread(img_file) height, width, _ = img_cv2.shape net.setInput(cv2.dnn.blobFromImage(img_cv2, size=(300, 300), swapRB=True, crop=False)) out = net.forward() print(out) for detection in out[0, 0, :,:]: score = float(detection[2]) if score > score_threshold: left = detection[3] * width top = detection[4] * height right = detection[5] * width bottom = detection[6] * height cv2.rectangle(img_cv2, (int(left), int(top)), (int(right), int(bottom)), (23, 230, 210), thickness=2) t, _ = net.getPerfProfile() label = 'Inference time: %.2f ms' % \ (t * 1000.0 / cv2.getTickFrequency()) cv2.putText(img_cv2, label, (0, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255)) plt.figure(figsize=(10, 8)) plt.imshow(img_cv2[:, :, ::-1]) plt.title("OpenCV DNN Faster RCNN-ResNet50") plt.axis("off") plt.show()3. Faster RCNN DNN 实现之二#!/usr/bin/python3 # -*- coding: utf-8 -*- import cv2 import os import matplotlib.pyplot as plt import time class general_faster_rcnn(object): def __init__(self, modelpath): self.conf_threshold = 0.3 # Confidence threshold self.nms_threshold = 0.4 # Non-maximum suppression threshold self.net_width = 416 # 300 # Width of network's input image self.net_height = 416 # 300 # Height of network's input image self.classes = self.get_coco_names() self.faster_rcnn_model = self.get_faster_rcnn_model(modelpath) self.outputs_names = self.get_outputs_names() def get_coco_names(self): classes = ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "background", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "background", "backpack", "umbrella", "background", "background", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "background", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "background", "dining table", "background", "background", "toilet", "background", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "background", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush", "background" ] return classes def get_faster_rcnn_model(self, modelpath): pb_file = os.path.join(modelpath, "frozen_inference_graph.pb") pbtxt_file = os.path.join(modelpath, "graph.pbtxt") net = cv2.dnn.readNetFromTensorflow(pb_file, pbtxt_file) net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) return net def get_outputs_names(self): # 网络中所有网络层的名字 layersNames = self.faster_rcnn_model.getLayerNames() # 网络输出层的名字，如，没有链接输出的网络层. return [layersNames[i[0] - 1] for i in \ self.faster_rcnn_model.getUnconnectedOutLayers()] # NMS 处理掉低 confidence 的边界框. def postprocess(self, img_cv2, outputs): img_height, img_width, _ = img_cv2.shape class_ids = [] confidences = [] boxes = [] for output in outputs: for detection in output[0, 0]: # [batch_id, class_id, confidence, left, top, right, bottom] confidence = detection[2] if confidence > self.conf_threshold: left = int(detection[3]*img_width) top = int(detection[4]*img_height) right = int(detection[5]*img_width) bottom = int(detection[6]*img_height) width = right - left + 1 height = bottom - top + 1 class_ids.append(int(detection[1])) confidences.append(float(confidence)) boxes.append([left, top, width, height]) # NMS 处理 indices = cv2.dnn.NMSBoxes(boxes, confidences, self.conf_threshold, self.nms_threshold) results = [] for ind in indices: res_box = {} res_box["class_id"] = class_ids[ind[0]] res_box["score"] = confidences[ind[0]] box = boxes[ind[0]] res_box["box"] = (box[0], box[1], box[0]+box[2], box[1]+box[3]) results.append(res_box) return results def predict(self, img_file): img_cv2 = cv2.imread(img_file) # 创建 4D blob. blob = cv2.dnn.blobFromImage( img_cv2, size=(self.net_width, self.net_height), swapRB=True, crop=False) # 设置网络的输入 blob self.faster_rcnn_model.setInput(blob) # 打印网络的输出层名 print("[INFO]Net output layers: {}".format(self.outputs_names)) # Runs forward outputs = self.faster_rcnn_model.forward(self.outputs_names) # NMS results = self.postprocess(img_cv2, outputs) return results def vis_res(self, img_file, results): img_cv2 = cv2.imread(img_file) for result in results: left, top, right, bottom = result["box"] cv2.rectangle(img_cv2, (left, top), (right, bottom), (255, 178, 50), 3) # Get the label for the class name and its confidence label = '%.2f' % result["score"] if self.classes: assert (result["class_id"] < len(self.classes)) label = '%s:%s' % (self.classes[result["class_id"]], label) label_size, baseline = cv2.getTextSize( label, cv2.FONT_HERSHEY_SIMPLEX, 0.5, 1) top = max(top, label_size[1]) cv2.rectangle( img_cv2, (left, top - round(1.5 * label_size[1])), (left + round(1.5 * label_size[0]), top + baseline), (255, 0, 0), cv2.FILLED) cv2.putText(img_cv2, label, (left, top), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 0), 1) t, _ = self.faster_rcnn_model.getPerfProfile() label = 'Inference time: %.2f ms' % \ (t * 1000.0 / cv2.getTickFrequency()) cv2.putText(img_cv2, label, (0, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255)) plt.figure(figsize=(10, 8)) plt.imshow(img_cv2[:,:,::-1]) plt.title("OpenCV DNN Faster RCNN-ResNet50") plt.axis("off") plt.show() if __name__ == '__main__': print("[INFO]Faster RCNN object detection in OpenCV.") img_file = "test.jpg" start = time.time() modelpath = "/path/to/faster_rcnn_resnet50_coco_2018_01_28/" faster_rcnn_model = general_faster_rcnn(modelpath) print("[INFO]Model loads time: ", time.time() - start) start = time.time() results = faster_rcnn_model.predict(img_file) print("[INFO]Model predicts time: ", time.time() - start) faster_rcnn_model.vis_res(img_file, results)网络输入为 (300, 300) 时，目标检测结果为(与 实现之一 中的结果一致.)：网络输入为 (416, 416) 时，目标检测结果为(提高输入图片分辨率有助于提升检测结果)：4. Faster RCNN TensorFlow 实现采用 TensorFlow 目标检测 API 进行模型测试：#!/usr/bin/python3 #!--*-- coding:utf-8 --*-- import os import numpy as np import cv2 import matplotlib.pyplot as plt import tensorflow as tf model_path = "/path/to/faster_rcnn_resnet50_coco_2018_01_28" frozen_pb_file = os.path.join(model_path, 'frozen_inference_graph.pb') score_threshold = 0.3 img_file = "test.jpg" # Read the graph. with tf.gfile.FastGFile(frozen_pb_file, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) with tf.Session() as sess: # Restore session sess.graph.as_default() tf.import_graph_def(graph_def, name='') # Read and preprocess an image. img_cv2 = cv2.imread(img_file) img_height, img_width, _ = img_cv2.shape img_in = cv2.resize(img_cv2, (416, 416)) img_in = img_in[:, :, [2, 1, 0]] # BGR2RGB # Run the model outputs = sess.run( [sess.graph.get_tensor_by_name('num_detections:0'), sess.graph.get_tensor_by_name('detection_scores:0'), sess.graph.get_tensor_by_name('detection_boxes:0'), sess.graph.get_tensor_by_name('detection_classes:0')], feed_dict={'image_tensor:0': img_in.reshape( 1, img_in.shape[0], img_in.shape[1], 3)}) # Visualize detected bounding boxes. num_detections = int(outputs[0][0]) for i in range(num_detections): classId = int(outputs[3][0][i]) score = float(outputs[1][0][i]) bbox = [float(v) for v in outputs[2][0][i]] if score > score_threshold: x = bbox[1] * img_width y = bbox[0] * img_height right = bbox[3] * img_width bottom = bbox[2] * img_height cv2.rectangle(img_cv2, (int(x), int(y)), (int(right), int(bottom)), (125, 255, 51), thickness=2) plt.figure(figsize=(10, 8)) plt.imshow(img_cv2[:, :, ::-1]) plt.title("TensorFlow Faster RCNN-ResNet50") plt.axis("off") plt.show()目标检测结果如：采用 TensorFlow 目标检测 API 对于相同的 (300, 300) 网络输入，得到的结果好像比 DNN 更好一些，原因暂未知.
在 OpenCV4.X 版本(OpenCV3.4.1之后版本) 可以采用 cv2.dnn.readNetFromTensorflow(pbmodel, pbtxt) 函数直接调用 TensorFlow 训练的目标检测模型.1. TensorFlow Detection Model ZooTensorFlow 目标检测预训练模型：Tensorflow Detection Model Zoo1.1. 基于 COCO 数据训练的模型Model nameSpeed (ms)COCO mAP1Outputsssd_mobilenet_v1_coco3021Boxesssd_mobilenet_v1_0.75_depth_coco ☆2618Boxesssd_mobilenet_v1_quantized_coco ☆2918Boxesssd_mobilenet_v1_0.75_depth_quantized_coco ☆2916Boxesssd_mobilenet_v1_ppn_coco ☆2620Boxesssd_mobilenet_v1_fpn_coco ☆5632Boxesssd_resnet_50_fpn_coco ☆7635Boxesssd_mobilenet_v2_coco3122Boxesssd_mobilenet_v2_quantized_coco2922Boxesssdlite_mobilenet_v2_coco2722Boxesssd_inception_v2_coco4224Boxesfaster_rcnn_inception_v2_coco5828Boxesfaster_rcnn_resnet50_coco8930Boxesfaster_rcnn_resnet50_lowproposals_coco64 Boxesrfcn_resnet101_coco9230Boxesfaster_rcnn_resnet101_coco10632Boxesfaster_rcnn_resnet101_lowproposals_coco82 Boxesfaster_rcnn_inception_resnet_v2_atrous_coco62037Boxesfaster_rcnn_inception_resnet_v2_atrous_lowproposals_coco241 Boxesfaster_rcnn_nas183343Boxesfaster_rcnn_nas_lowproposals_coco540 Boxesmask_rcnn_inception_resnet_v2_atrous_coco77136Masksmask_rcnn_inception_v2_coco7925Masksmask_rcnn_resnet101_atrous_coco47033Masksmask_rcnn_resnet50_atrous_coco34329Masks注:[1] - 带五角星符号(☆) 表示模型支持 TPU 训练.[2] - 下载 quantized 模型的 .tar.gz 文件并解压后，会得到不同的文件，包括：checkpoint 文件，config 配置文件和 tfile frozen graphs(txt/binary)文件.1.2. 基于 Kitti 数据集训练的模型Model nameSpeed (ms)Pascal mAP@0.5Outputsfaster_rcnn_resnet101_kitti7987Boxes1.3. 基于 Open Images 数据集训练的模型Model nameSpeed (ms)Open Images mAP@0.52Outputsfaster_rcnn_inception_resnet_v2_atrous_oidv272737Boxesfaster_rcnn_inception_resnet_v2_atrous_lowproposals_oidv2347 Boxesfacessd_mobilenet_v2_quantized_open_image_v4 32073 (faces)BoxesModel nameSpeed (ms)Open Images mAP@0.54Outputsfaster_rcnn_inception_resnet_v2_atrous_oidv442554Boxesssd_mobilenetv2_oidv48936Boxesssd_resnet_101_fpn_oidv423738Boxes1.4. 基于 iNaturalist Species 数据集训练的模型Model nameSpeed (ms)Pascal mAP@0.5Outputsfaster_rcnn_resnet101_fgvc39558Boxesfaster_rcnn_resnet50_fgvc36655Boxes1.5. 基于 AVA v2.1 训练的模型Model nameSpeed (ms)Pascal mAP@0.5Outputsfaster_rcnn_resnet101_ava_v2.19311Boxes1.6. TensorFlow 目标检测 API - SSD 例示TensorFlow 中，深度学习网络被表示为图(graphs)，其中图中每个节点(node) 是其输入的一种变换. 节点可以是常用网络层，如 C++ 实现的 Convolution 和 MaxPooling 层. 也可以采用 python 利用 TensorFlow 操作子(operations) 来构建自定义网络层.TensorFlow 目标检测API 是用于创建目标检测深度网络的框架. TensorFlow 训练得到的模型是 .pb 后缀的二值文件，其同时保存了训练网络的拓扑(topology)结构和模型权重. 这里以 ssd_mobilenet_v2_coco_2018_03_29 预训练模型(基于 COCO 数据集训练的 MobileNet-SSD模型)为例：#!/usr/bin/python3 #!--*-- coding:utf-8 --*-- import os import numpy as np import cv2 import matplotlib.pyplot as plt import tensorflow as tf model_path = "/path/to/ssd_mobilenet_v2_coco_2018_03_29" frozen_pb_file = os.path.join(model_path, 'frozen_inference_graph.pb') score_threshold = 0.3 img_file = 'test.jpg' # Read the graph. with tf.gfile.FastGFile(frozen_pb_file, 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) with tf.Session() as sess: # Restore session sess.graph.as_default() tf.import_graph_def(graph_def, name='') # Read and preprocess an image. img_cv2 = cv2.imread(img_file) img_height, img_width, _ = img_cv2.shape img_in = cv2.resize(img_cv2, (300, 300)) img_in = img_in[:, :, [2, 1, 0]] # BGR2RGB # Run the model outputs = sess.run([sess.graph.get_tensor_by_name('num_detections:0'), sess.graph.get_tensor_by_name('detection_scores:0'), sess.graph.get_tensor_by_name('detection_boxes:0'), sess.graph.get_tensor_by_name('detection_classes:0')], feed_dict={ 'image_tensor:0': img_in.reshape(1, img_in.shape[0], img_in.shape[1], 3)}) # Visualize detected bounding boxes. num_detections = int(outputs[0][0]) for i in range(num_detections): classId = int(outputs[3][0][i]) score = float(outputs[1][0][i]) bbox = [float(v) for v in outputs[2][0][i]] if score > score_threshold: x = bbox[1] * img_width y = bbox[0] * img_height right = bbox[3] * img_width bottom = bbox[2] * img_height cv2.rectangle(img_cv2, (int(x), int(y)), (int(right), int(bottom)), (125, 255, 51), thickness=2) plt.figure(figsize=(10, 8)) plt.imshow(img_cv2[:, :, ::-1]) plt.title("TensorFlow MobileNetV2-SSD") plt.axis("off") plt.show()目标检测结果如：https://github.com/opencv/opencv/wiki/TensorFlow-Object-Detection-API2. TensorFlow 目标检测模型转换为 DNN 可调用格式OpenCV DNN 模块调用 TensorFlow 训练的目标检测模型时，需要一个额外的配置文件，其主要是基于与 protocol buffers(protobuf) 格式序列化图(graph) 相同的文本格式版本.2.1. DNN 已可直接调用检测模型OpenCV 中已经提供的 TensorFlow 目标检测模型和配置文件有：ModelVersion MobileNet-SSD v12017_11_17weightsconfigMobileNet-SSD v1 PPN2018_07_03weightsconfigMobileNet-SSD v22018_03_29weightsconfigInception-SSD v22017_11_17weightsconfigFaster-RCNN Inception v22018_01_28weightsconfigFaster-RCNN ResNet-502018_01_28weightsconfigMask-RCNN Inception v22018_01_28weightsconfig2.2. 常用目标检测模型转换三种不同的 TensorFlow 目标检测模型转换脚本为：tf_text_graph_ssd.pytf_text_graph_faster_rcnn.pytf_text_graph_mask_rcnn.pyFrom: https://github.com/opencv/opencv/tree/master/samples/dnn转换脚本的输入参数：[1] - --input: TensorFlow frozen graph 文件路径.[2] - --config: TensorFlow 模型训练时的 *.config 文件路径.注： TensorFlow *.config配置文件：configuration file.转换脚本的输出参数：[1] - --output: 输出的 text graph 文件.如：faster rcnn 模型： python3 tf_text_graph_faster_rcnn.py \ --input '/path/to/faster_rcnn_resnet50_coco_2018_01_28/frozen_inference_graph.pb' \ --config '/path/to/faster_rcnn_resnet50_coco.config' \ --output '/path/to/faster_rcnn_resnet50_coco_2018_01_28/graph.pbtxt' ssd 模型：python3 tf_text_graph_ssd.py \ --input /path/to/ssd_inception_v2_coco_2018_01_28/frozen_inference_graph.pb \ --config /path/to/ssd_inception_v2_coco.config \ --output /path/to/ssd_inception_v2_coco_2018_01_28/graph.pbtxtmask rcnn 模型：python3 tf_text_graph_mask_rcnn2.py \ --input '/path/to/mask_rcnn_resnet50_atrous_coco_2018_01_28/frozen_inference_graph.pb' \ --config '/path/to/mask_rcnn_resnet50_atrous_coco.config' \ --output '/path/to/mask_rcnn_resnet50_atrous_coco_2018_01_28/graph.pbtxt' 对于生成的 graph.pbtxt 可采用 Netron 工具进行可视化.Github 项目 - 深度网络可视化工具 - Netron - AIUAI2.3. DNN 目标检测 - SSD 例示与 TensorFLow 目标检测 API -SSD 例示 一样，检测测试下基于 OpenCV DNN 的 SSD 目标检测.[1] - 首先进行模型转换，如：python3 tf_text_graph_ssd.py \ --input '/path/to/ssd_mobilenet_v2_coco_2018_03_29/frozen_inference_graph.pb' \ --config '/path/to/ssd_mobilenet_v2_coco.config' \ --output '/path/to/ssd_mobilenet_v2_coco_2018_03_29/graph.pbtxt' 终端打印的转换过程如：Scale: [0.200000-0.950000] Aspect ratios: [1.0, 2.0, 0.5, 3.0, 0.3333] Reduce boxes in the lowest layer: True Number of classes: 90 Number of layers: 6 box predictor: convolutional Input image size: 300x300[2] - 然后，目标检测模型测试：#!/usr/bin/python #!--*-- coding:utf-8 --*-- import cv2 import matplotlib.pyplot as plt pb_file = '/path/to/ssd_inception_v2_coco_2018_01_28/frozen_inference_graph.pb' pbtxt_file = '/path/to/ssd_inception_v2_coco_2018_01_28/graph.pbtxt' net = cv2.dnn.readNetFromTensorflow(pb_file, pbtxt_file) score_threshold = 0.3 img_file = 'test.jpg' img_cv2 = cv2.imread(img_file) height, width, _ = img_cv2.shape net.setInput(cv2.dnn.blobFromImage(img_cv2, size=(300, 300), swapRB=True, crop=False)) out = net.forward() print(out) for detection in out[0, 0, :,:]: score = float(detection[2]) if score > score_threshold: left = detection[3] * width top = detection[4] * height right = detection[5] * width bottom = detection[6] * height cv2.rectangle(img_cv2, (int(left), int(top)), (int(right), int(bottom)), (23, 230, 210), thickness=2) t, _ = net.getPerfProfile() label = 'Inference time: %.2f ms' % \ (t * 1000.0 / cv2.getTickFrequency()) cv2.putText(img_cv2, label, (0, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255)) plt.figure(figsize=(10, 8)) plt.imshow(img_cv2[:, :, ::-1]) plt.title("OpenCV DNN MobileNetV2-SSD") plt.axis("off") plt.show()目标检测结果如：1 ↩2 ↩3 ↩4 ↩
OpenPose 可以对图片中单个人体目标的姿态估计，也可以处理图片中多人的姿态估计.OpenPose 基于OpenCV DNN 的单人姿态估计 - AIUAI这里主要记录基于 OpenCV 4.x DNN 模块和 OpenPose 开源模型的多人人体姿态估计 的实现.采用的模型的基于 COCO 数据集训练的人体关键点模型.1. OpenPose 网络结构OpenPose 的多人人体姿态估计的模型结构如图：网络输入为 hxwx3 的图片，并输出包含关键点(Keypoints)的置信图(confidence maps) 和每个关键点对(keypoint pair) 的 Part Affinity Heatmaps 的两个输出数组.[1] - Stage 0:采用 VGGNet 的前 10 层，提取输入图片的特征图(feature maps).[2] - Stage 1:采用 2-分支的 multi-stage CNN 网络结构：(1) - 分支一：网络分支一预测人体关键点位置的 2D Confidence Maps ，如 elbow, knee, etc. 每个 Confidence Map 是一个灰度图(grayscale image)，其最大值的位置坐标即为对应人体某个关键点的概率最高. 如图：(2) - 分支二：网络分支二预测 Part Affinities (PAF) 的2D 向量场(L, vector fields)，其表示了两个关键点之间关联度(degree of association). 例如，关键点 Neck 和 Left Shoulder 之间的 Part Affinity, 如下图，属于同一个人体的关键点之间的 Affinity 值比较大.总体来说，Confidence Maps 用于检测关键点位置；而 Affinity Maps 用于检测关键点之间的有效连接.2. OpenPose 的 OpenCV DNN 实现根据检测过程，主要涉及的函数有：getKeyponts() 、 getValidPairs() 和 getPersonwiseKeypoints() .2.1. 模型加载#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import cv2 import time import numpy as np import matplotlib.pyplot as plt num_points = 18 keypointsMapping = ['Nose', 'Neck', 'R-Sho', 'R-Elb', 'R-Wr', 'L-Sho', 'L-Elb', 'L-Wr', 'R-Hip', 'R-Knee', 'R-Ank', 'L-Hip', 'L-Knee', 'L-Ank', 'R-Eye', 'L-Eye', 'R-Ear', 'L-Ear'] POSE_PAIRS = [[1,2], [1,5], [2,3], [3,4], [5,6], [6,7], [1,8], [8,9], [9,10], [1,11], [11,12], [12,13], [1,0], [0,14], [14,16], [0,15], [15,17], [2,17], [5,16] ] # index of pafs correspoding to the POSE_PAIRS # e.g for POSE_PAIR(1,2), the PAFs are located at indices (31,32) of output, Similarly, (1,5) -> (39,40) and so on. mapIdx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], [23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], [55,56], [37,38], [45,46]] colors = [[0,100,255], [0,100,255], [0,255,255], [0,100,255], [0,255,255], [0,100,255], [0,255,0], [255,200,100], [255,0,255], [0,255,0], [255,200,100], [255,0,255], [0,0,255], [255,0,0], [200,200,0], [255,0,0], [200,200,0], [0,0,0]] # dnn 加载模型 start = time.time() prototxt = "./models/pose/coco/pose_deploy_linevec.prototxt" caffemodel = "./models/pose/coco/pose_iter_440000.caffemodel" net = cv2.dnn.readNetFromCaffe(prototxt, caffemodel) print("[INFO]Time Taken in Model Loading: {}".format(time.time() - start)) # test img img_cv2 = cv2.imread("test_image.jpg") img_width, img_height = img_cv2.shape[1], img_cv2.shape[0] # 根据长宽比，固定网路输入的 height，计算网络输入的 width. net_height = 368 net_width = int((net_height/img_height)*img_width) start = time.time() in_blob = cv2.dnn.blobFromImage( img_cv2, 1.0 / 255, (net_width, net_height), (0, 0, 0), swapRB=False, crop=False) net.setInput(in_blob) output = net.forward() print("[INFO]Time Taken in Forward pass: {}".format(time.time() - start))2.2. 关键点检测2.2.1. getKeyponts( )函数getKeyponts( )函数功能：对 Confidence Map 采用 NMS(Non Maximum Suppression) 来检测关键点.def getKeypoints(probMap, threshold=0.1): mapSmooth = cv2.GaussianBlur(probMap,(3,3),0,0) mapMask = np.uint8(mapSmooth>threshold) keypoints = [] # 1. 找出对应于关键点的所有区域的轮廓(contours) contours, hierarchy = cv2.findContours( mapMask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # for each blob find the maxima # 对于每个关键点轮廓区域，找到最大值. for cnt in contours: # 2. 创建关键点的 mask； blobMask = np.zeros(mapMask.shape) blobMask = cv2.fillConvexPoly(blobMask, cnt, 1) # 3. 提取关键点区域的 probMap maskedProbMap = mapSmooth * blobMask # 4. 提取关键点区域的局部最大值. _, maxVal, _, maxLoc = cv2.minMaxLoc(maskedProbMap) keypoints.append(maxLoc + (probMap[maxLoc[1], maxLoc[0]],)) return keypoints2.2.2. 关键点 mapMaskthreshold = 0.1 part = 1 probMap = output[0, part, :,:] mapSmooth = cv2.GaussianBlur(probMap, (3, 3), 0, 0) mapMask = np.uint8(mapSmooth > threshold) plt.subplot(1, 3, 1) plt.imshow(probMap) plt.title("probMap") plt.axis("off") plt.subplot(1, 3, 2) plt.imshow(mapSmooth) plt.title("mapSmooth") plt.axis("off") plt.subplot(1, 3, 3) plt.imshow(mapMask) plt.title("mapMask") plt.axis("off") plt.show()2.2.3. 关键点坐标值threshold = 0.1 part = 1 probMap = output[0, part, :,:] probMap = cv2.resize(probMap, (img_cv2.shape[1], img_cv2.shape[0])) mapSmooth = cv2.GaussianBlur(probMap, (3, 3), 0, 0) mapMask = np.uint8(mapSmooth > threshold) keypoints = [] contours, hierarchy = cv2.findContours(mapMask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) for cnt in contours: blobMask = np.zeros(mapMask.shape) blobMask = cv2.fillConvexPoly(blobMask, cnt, 1) maskedProbMap = mapSmooth * blobMask _, maxVal, _, maxLoc = cv2.minMaxLoc(maskedProbMap) keypoints.append(maxLoc + (probMap[maxLoc[1], maxLoc[0]],)) img_cv2_copy = img_cv2.copy() for keypoint in keypoints: cv2.circle(img_cv2_copy, (keypoint[0], keypoint[1]), 5, (255, 0, 0), -1, cv2.LINE_AA) plt.figure() plt.imshow(img_cv2_copy[:, :, ::-1]) plt.title("Keypoints") plt.axis("off") plt.show()2.3. 有效关键点对检测2.3.1. getValidPairs()函数getValidPairs()函数功能：检测所有人体之间不同关键点之间的有效连接.计算公式为：$$ E = \int_{u=0}^{u=1} \mathbf{L}_c (\mathbf{p}(u)) \cdot \frac{\mathbf{d}_{j2} - \mathbf{d}_{j1}}{||\mathbf{d}_{j2} - \mathbf{d}_{j1}||_2} du $$其中，$\mathbf{p}(u)$ 是对两个关键点 $\mathbf{d}_{j1}$ 和 $\mathbf{d}_{j2}$ 之间的位置进行插值：$$ \mathbf{p}(u) = (1 - u)\mathbf{d}_{j1} + u\mathbf{d}_{j2} $$$\mathbf{L}$ 是Part Affinity(PAF)；$\mathbf{d}$ 是两个关键点的向量.$\mathbf{p}$ 是两个关键点之间的插值点.$\mathbf{E}$ 是通过计算 PAF $\mathbf{L}$ 和向量 $d_{ij}$ 之间的点积(dot product) 得到的.# Find valid connections between the different # joints of a all persons present def getValidPairs(output): valid_pairs = [] invalid_pairs = [] n_interp_samples = 10 paf_score_th = 0.1 conf_th = 0.7 # loop for every POSE_PAIR for k in range(len(mapIdx)): # A->B constitute a limb pafA = output[0, mapIdx[k][0], :, :] pafB = output[0, mapIdx[k][1], :, :] pafA = cv2.resize(pafA, (imgWidth, imgHeight)) pafB = cv2.resize(pafB, (imgWidth, imgHeight)) # 检测第一个 limb 和第二个 limb 的关键点位置 candA = detected_keypoints[POSE_PAIRS[k][0]] candB = detected_keypoints[POSE_PAIRS[k][1]] nA = len(candA) nB = len(candB) # 如果检测到 joint-pair 的关键点位置，则， # 检查 candA 和 candB 中每个 joint. # 计算两个 joints 之间的距离向量(distance vector). # 计算两个 joints 之间插值点集合的 PAF 值. # 根据上述公式，计算 score 值，判断连接的有效性. if( nA != 0 and nB != 0): valid_pair = np.zeros((0,3)) for i in range(nA): max_j=-1 maxScore = -1 found = 0 for j in range(nB): # Find d_ij d_ij = np.subtract(candB[j][:2], candA[i][:2]) norm = np.linalg.norm(d_ij) if norm: d_ij = d_ij / norm else: continue # Find p(u) interp_coord = list( zip(np.linspace(candA[i][0], candB[j][0], num=n_interp_samples), np.linspace(candA[i][1], candB[j][1], num=n_interp_samples))) # Find L(p(u)) paf_interp = [] for k in range(len(interp_coord)): paf_interp.append( [pafA[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))], pafB[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))] ]) # Find E paf_scores = np.dot(paf_interp, d_ij) avg_paf_score = sum(paf_scores)/len(paf_scores) # 判断连接是否有效. # 如果对应于 PAF 的插值向量值大于阈值，则连接有效. if (len(np.where(paf_scores > paf_score_th)[0])/ n_interp_samples ) > conf_th : if avg_paf_score > maxScore: max_j = j maxScore = avg_paf_score found = 1 # Append the connection to the list if found: valid_pair = np.append( valid_pair, [[candA[i][3], candB[max_j][3], maxScore]], axis=0) # Append the detected connections to the global list valid_pairs.append(valid_pair) else: # If no keypoints are detected print("No Connection : k = {}".format(k)) invalid_pairs.append(k) valid_pairs.append([]) print(valid_pairs) return valid_pairs, invalid_pairs2.3.2. 关键点对检测具体实现分析有效的关键点对(joint pair) 是指两个关键点的连接，属于相同的人体.最简单的方式是，计算一个关键点与其它所有可能的关键点之间的最小距离，来判断关键点对的有效性. 例如下图，可以计算 Nose 关键点与其它所有 Necks 关键点之间的距离，最小距离的 Neck 关键点，则对应于同一个人体.但是，这种方法不是对所有的关键点对都是有效的，尤其是对与图像中包含很多人体，或者有关键点缺失的时候. 例如下图，对于关键点对， Left-Elbow -> Left Wrist，第三个人体的 wrist 关键点与第二个人体的 elbow 关键点的距离，比与其同一人体的 elbow 关键点的距离更小. 但该关键点对并不是有效的.而 Part Affinity Maps 的作用是，给定沿着两个关键点对的仿射(affinity)的方向. 因此，有效的关键点对不仅具有最小的距离，其方向也应该顺着 PAF Heatmaps 方向.例如，Left-Elbow -> Left Wrist 连接的 Heatmap，如下图，即使最小距离的关键点是错误的，但，由于 PAF 只能顺着 Elbow 和 Wrist 的单位向量，所以能正确检测有效的关键点对：OpenPose 中采用的方法为：[1] - 将关键点对的两个点之间的连线进行划分，得到该连线上的 n 个点(Divide the line joining the two points comprising the pair. Find n points on this line.)；[2] - 判断这些点上的 PAF 是否与连接该关键点的线的方向相同(Check if the PAF on these points have the same direction as that of the line joining the points for this pair)；[3] - 如果方向满足特定程度，则为有效的关键点对(If the direction matches to a certain extent, then it is valid pair.)代码实现与分析：对于每个关键点对(body part pair, keypoints pair)，[1] - 选择属于同一个关键点对的关键点. 并分别存放在两个列表: candA 和 candB. candA 列表中的每个关键点可以与 candB 中的某些关键点相连接. 如下图，给出了 candA 和 candB 中的 Neck -> Right-Shoulder 关键点对的所有关键点：对应 Python 实现：pafA = output[0, mapIdx[k][0], :, :] pafB = output[0, mapIdx[k][1], :, :] pafA = cv2.resize(pafA, (frameWidth, frameHeight)) pafB = cv2.resize(pafB, (frameWidth, frameHeight)) # Find the keypoints for the first and second limb candA = detected_keypoints[POSE_PAIRS[k][0]] candB = detected_keypoints[POSE_PAIRS[k][1]][2] - 计算两个关键点之间的单位向量，其给定了关节点之间连线的方向.d_ij = np.subtract(candB[j][:2], candA[i][:2]) norm = np.linalg.norm(d_ij) if norm: d_ij = d_ij / norm[3] - 计算两个关键点之间连线的 10 个插值点.# Find p(u) interp_coord = list(zip(np.linspace(candA[i][0], candB[j][0], num=n_interp_samples), np.linspace(candA[i][1], candB[j][1], num=n_interp_samples))) # Find L(p(u)) paf_interp = [] for k in range(len(interp_coord)): paf_interp.append( [pafA[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))], pafB[int(round(interp_coord[k][1])), int(round(interp_coord[k][0]))] ]) [4] - 计算插值点的 PAF 与单位向量 d_ij 之间的点积(dot product).# Find E paf_scores = np.dot(paf_interp, d_ij) avg_paf_score = sum(paf_scores)/len(paf_scores)[5] - 如果这些插值点的 70% 的都满足判定标准，则该关键点对是有效的.# Check if the connection is valid # If the fraction of interpolated vectors # aligned with PAF is higher then threshold -> Valid Pair if (len(np.where(paf_scores > paf_score_th)[0])/n_interp_samples ) > conf_th : if avg_paf_score > maxScore: max_j = j maxScore = avg_paf_score2.4. 同一人体关键点的组合2.4.1. getPersonwiseKeypoints() 函数getPersonwiseKeypoints()函数功能：计算得到属于每个人体的关键点集合.对于每个检测到的有效 joint pair，分配属于一个人体的 joints.def getPersonwiseKeypoints(valid_pairs, invalid_pairs): # 每一行的最后一个值为 overall score. personwiseKeypoints = -1 * np.ones((0, 19)) for k in range(len(mapIdx)): if k not in invalid_pairs: partAs = valid_pairs[k][:,0] partBs = valid_pairs[k][:,1] indexA, indexB = np.array(POSE_PAIRS[k]) for i in range(len(valid_pairs[k])): found = 0 person_idx = -1 for j in range(len(personwiseKeypoints)): if personwiseKeypoints[j][indexA] == partAs[i]: person_idx = j found = 1 break if found: personwiseKeypoints[person_idx][indexB] = partBs[i] personwiseKeypoints[person_idx][-1] += keypoints_list[partBs[i].astype(int), 2] + valid_pairs[k][i][2] # if find no partA in the subset, create a new subset elif not found and k < 17: row = -1 * np.ones(19) row[indexA] = partAs[i] row[indexB] = partBs[i] # add the keypoint_scores for the two keypoints and the paf_score row[-1] = sum( keypoints_list[valid_pairs[k][i,:2].astype(int), 2]) + valid_pairs[k][i][2] personwiseKeypoints = np.vstack([personwiseKeypoints, row]) return personwiseKeypoints2.4.2. 关键点组合的具体实现分析计算得到所有关键点之间的关键点对后，可以将具有相同关键点检测候选值的关键点对，组合为多人的姿态估计.代码实现具体分析[1] - 首先创建保存每个人体的所有关键点的空列表. 然后对每个关键点对，判断 partA 是否已经在列表里, 如果已经在列表里，则表示该关键点对属于该列表，且 partB 也属于同一人体. 因此，添加该关键点对的 partB 到 partA 所在的列表.# 空列表 personwiseKeypoints = -1 * np.ones((0, 19))for j in range(len(personwiseKeypoints)): if personwiseKeypoints[j][indexA] == partAs[i]: person_idx = j found = 1 break if found: personwiseKeypoints[person_idx][indexB] = partBs[i][2] - 如果 partA 不在任一人体列表里，则表示该关键点对属于一个新出现的人体，故创建新的列表.# if find no partA in the subset, create a new subset elif not found and k < 17: row = -1 * np.ones(19) row[indexA] = partAs[i] row[indexB] = partBs[i]2.5. 姿态检测结果可视化## img_cv2_copy = img_cv2.copy() for i in range(num_points): for j in range(len(detected_keypoints[i])): cv2.circle(img_cv2_copy, detected_keypoints[i][j][0:2], 5, colors[i], -1, cv2.LINE_AA) for i in range(17): for n in range(len(personwiseKeypoints)): index = personwiseKeypoints[n][np.array(POSE_PAIRS[i])] if -1 in index: continue B = np.int32(keypoints_list[index.astype(int), 0]) A = np.int32(keypoints_list[index.astype(int), 1]) cv2.line(img_cv2_copy, (B[0], A[0]), (B[1], A[1]), colors[i], 3, cv2.LINE_AA) plt.figure() plt.imshow(img_cv2_copy[:, :, ::-1]) plt.title("Results") plt.axis("off") plt.show()如：3. 完整实现模型下载：COCO: http://posefs1.perception.cs.cmu.edu/OpenPose/models/pose/coco/pose_iter_440000.caffemodel#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import os import cv2 import time import numpy as np import matplotlib.pyplot as plt class general_mulitpose_model(object): def __init__(self): self.point_names = ['Nose', 'Neck', 'R-Sho', 'R-Elb', 'R-Wr', 'L-Sho', 'L-Elb', 'L-Wr', 'R-Hip', 'R-Knee', 'R-Ank', 'L-Hip', 'L-Knee', 'L-Ank', 'R-Eye', 'L-Eye', 'R-Ear', 'L-Ear'] self.point_pairs = [[1,2], [1,5], [2,3], [3,4], [5,6], [6,7], [1,8], [8,9], [9,10], [1,11], [11,12], [12,13], [1,0], [0,14], [14,16], [0,15], [15,17], [2,17], [5,16] ] # index of pafs correspoding to the self.point_pairs # e.g for point_pairs(1,2), the PAFs are located at indices (31,32) of output, # Similarly, (1,5) -> (39,40) and so on. self.map_idx = [[31,32], [39,40], [33,34], [35,36], [41,42], [43,44], [19,20], [21,22], [23,24], [25,26], [27,28], [29,30], [47,48], [49,50], [53,54], [51,52], [55,56], [37,38], [45,46]] self.colors = [[0,100,255], [0,100,255], [0,255,255], [0,100,255], [0,255,255], [0,100,255], [0,255,0], [255,200,100], [255,0,255], [0,255,0], [255,200,100], [255,0,255], [0,0,255], [255,0,0], [200,200,0], [255,0,0], [200,200,0], [0,0,0]] self.num_points = 18 self.pose_net = self.get_model() def get_model(self): prototxt = "./models/pose/coco/pose_deploy_linevec.prototxt" caffemodel = "./models/pose/coco/pose_iter_440000.caffemodel" coco_net = cv2.dnn.readNetFromCaffe(prototxt, caffemodel) return coco_net def getKeypoints(self, probMap, threshold=0.1): mapSmooth = cv2.GaussianBlur(probMap, (3, 3), 0, 0) mapMask = np.uint8(mapSmooth > threshold) keypoints = [] # find the blobs contours, hierarchy = cv2.findContours(mapMask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # for each blob find the maxima for cnt in contours: blobMask = np.zeros(mapMask.shape) blobMask = cv2.fillConvexPoly(blobMask, cnt, 1) maskedProbMap = mapSmooth * blobMask _, maxVal, _, maxLoc = cv2.minMaxLoc(maskedProbMap) keypoints.append(maxLoc + (probMap[maxLoc[1], maxLoc[0]],)) return keypoints def getValidPairs(self, output, detected_keypoints, img_width, img_height): valid_pairs = [] invalid_pairs = [] n_interp_samples = 10 paf_score_th = 0.1 conf_th = 0.7 for k in range(len(self.map_idx)): # A->B constitute a limb pafA = output[0, self.map_idx[k][0], :, :] pafB = output[0, self.map_idx[k][1], :, :] pafA = cv2.resize(pafA, (img_width, img_height)) pafB = cv2.resize(pafB, (img_width, img_height)) # Find the keypoints for the first and second limb candA = detected_keypoints[self.point_pairs[k][0]] candB = detected_keypoints[self.point_pairs[k][1]] nA = len(candA) nB = len(candB) if (nA != 0 and nB != 0): valid_pair = np.zeros((0, 3)) for i in range(nA): max_j = -1 maxScore = -1 found = 0 for j in range(nB): # Find d_ij d_ij = np.subtract(candB[j][:2], candA[i][:2]) norm = np.linalg.norm(d_ij) if norm: d_ij = d_ij / norm else: continue # Find p(u) interp_coord = list( zip(np.linspace(candA[i][0], candB[j][0], num=n_interp_samples), np.linspace(candA[i][1], candB[j][1], num=n_interp_samples))) # Find L(p(u)) paf_interp = [] for k in range(len(interp_coord)): paf_interp.append([pafA[int(round(interp_coord[k][1])), int( round(interp_coord[k][0]))], pafB[int(round(interp_coord[k][1])), int( round(interp_coord[k][0]))]]) # Find E paf_scores = np.dot(paf_interp, d_ij) avg_paf_score = sum(paf_scores) / len(paf_scores) # Check if the connection is valid if (len(np.where(paf_scores > paf_score_th)[ 0]) / n_interp_samples) > conf_th: if avg_paf_score > maxScore: max_j = j maxScore = avg_paf_score found = 1 # Append the connection to the list if found: valid_pair = np.append(valid_pair, [[candA[i][3], candB[max_j][3], maxScore]], axis=0) # Append the detected connections to the global list valid_pairs.append(valid_pair) else: # If no keypoints are detected print("No Connection : k = {}".format(k)) invalid_pairs.append(k) valid_pairs.append([]) return valid_pairs, invalid_pairs def getPersonwiseKeypoints(self, valid_pairs, invalid_pairs, keypoints_list): personwiseKeypoints = -1 * np.ones((0, 19)) for k in range(len(self.map_idx)): if k not in invalid_pairs: partAs = valid_pairs[k][:, 0] partBs = valid_pairs[k][:, 1] indexA, indexB = np.array(self.point_pairs[k]) for i in range(len(valid_pairs[k])): found = 0 person_idx = -1 for j in range(len(personwiseKeypoints)): if personwiseKeypoints[j][indexA] == partAs[i]: person_idx = j found = 1 break if found: personwiseKeypoints[person_idx][indexB] = partBs[i] personwiseKeypoints[person_idx][-1] += keypoints_list[ partBs[i].astype(int), 2] + \ valid_pairs[k][i][2] # if find no partA in the subset, create a new subset elif not found and k < 17: row = -1 * np.ones(19) row[indexA] = partAs[i] row[indexB] = partBs[i] # add the keypoint_scores for the two keypoints and the paf_score row[-1] = sum(keypoints_list[valid_pairs[k][i, :2].astype(int), 2]) + \ valid_pairs[k][i][2] personwiseKeypoints = np.vstack([personwiseKeypoints, row]) return personwiseKeypoints def predict(self, inputparam): img_cv2 = cv2.imread(inputparam["filename"]) img_width, img_height = img_cv2.shape[1], img_cv2.shape[0] net_height = 368 net_width = int((net_height / img_height) * img_width) start = time.time() in_blob = cv2.dnn.blobFromImage( img_cv2, 1.0 / 255, (net_width, net_height), (0, 0, 0), swapRB=False, crop=False) self.pose_net.setInput(in_blob) output = self.pose_net.forward() print("[INFO]Time Taken in Forward pass: {}".format(time.time() - start)) detected_keypoints = [] keypoints_list = np.zeros((0, 3)) keypoint_id = 0 threshold = 0.1 for part in range(self.num_points): probMap = output[0, part, :, :] probMap = cv2.resize(probMap, (img_cv2.shape[1], img_cv2.shape[0])) keypoints = self.getKeypoints(probMap, threshold) print("Keypoints - {} : {}".format(self.point_names[part], keypoints)) keypoints_with_id = [] for i in range(len(keypoints)): keypoints_with_id.append(keypoints[i] + (keypoint_id,)) keypoints_list = np.vstack([keypoints_list, keypoints[i]]) keypoint_id += 1 detected_keypoints.append(keypoints_with_id) valid_pairs, invalid_pairs = \ self.getValidPairs(output, detected_keypoints, img_width, img_height) personwiseKeypoints = \ self.getPersonwiseKeypoints(valid_pairs, invalid_pairs, keypoints_list) return personwiseKeypoints, keypoints_list def vis_pose(self, img_file, personwiseKeypoints, keypoints_list): img_cv2 = cv2.imread(img_file) for i in range(17): for n in range(len(personwiseKeypoints)): index = personwiseKeypoints[n][np.array(self.point_pairs[i])] if -1 in index: continue B = np.int32(keypoints_list[index.astype(int), 0]) A = np.int32(keypoints_list[index.astype(int), 1]) cv2.line(img_cv2, (B[0], A[0]), (B[1], A[1]), self.colors[i], 3, cv2.LINE_AA) plt.figure() plt.imshow(img_cv2[:, :, ::-1]) plt.title("Results") plt.axis("off") plt.show() if __name__ == '__main__': print("[INFO]MultiPose estimation.") img_file = "multipose_test_image.jpg" start = time.time() multipose_model = general_mulitpose_model() print("[INFO]Time Taken in Model Loading: {}".\ format(time.time() - start)) personwiseKeypoints, keypoints_list = \ multipose_model.predict(inputparam) multipose_model.vis_pose(img_file, personwiseKeypoints, keypoints_list) print(personwiseKeypoints) print(keypoints_list) print("[INFO]Done.")如：4. 参考资料[1] - Multi-Person Pose Estimation in OpenCV using OpenPose - 2018.09.11[2] - Github - spmallick/learnopencv/OpenPose-Multi-Person
OpenCV4.0 版本以后可以直接读取 Caffe、TensorFlow、ONNX 等模型的 API，直接采用OpenCV 的 DNN 模块即可.这里主要测试下基于 DNN 模块和 OpenPose 模型的单人人体姿态估计的具体实现.Github 项目 - OpenPose 关键点输出格式 - AIUAIGithub 项目 - OpenPose Python API - AIUAIGithub 项目 - OpenPose 模型与Demos - AIUAIOpenPose 人体姿态模型下载路径：BODY25: http://posefs1.perception.cs.cmu.edu/OpenPose/models/pose/body_25/pose_iter_584000.caffemodel COCO: http://posefs1.perception.cs.cmu.edu/OpenPose/models/pose/coco/pose_iter_440000.caffemodel MPI: http://posefs1.perception.cs.cmu.edu/OpenPose/models/pose/mpi/pose_iter_160000.caffemodel具体完整代码为：#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- from __future__ import division import cv2 import time import numpy as np import matplotlib.pyplot as plt import os class general_pose_model(object): def __init__(self, modelpath, mode="BODY25"): # 指定采用的模型 # Body25: 25 points # COCO: 18 points # MPI: 15 points self.inWidth = 368 self.inHeight = 368 self.threshold = 0.1 if mode == "BODY25": self.pose_net = self.general_body25_model(modelpath) elif mode == "COCO": self.pose_net = self.general_coco_model(modelpath) elif mode == "MPI": self.pose_net = self.get_mpi_model(modelpath) def get_mpi_model(self, modelpath): self.points_name = { "Head": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4, "LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9, "RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "Chest": 14, "Background": 15 } self.num_points = 15 self.point_pairs = [[0, 1], [1, 2], [2, 3], [3, 4], [1, 5], [5, 6], [6, 7], [1, 14],[14, 8], [8, 9], [9, 10], [14, 11], [11, 12], [12, 13] ] prototxt = os.path.join( modelpath, "pose/mpi/pose_deploy_linevec_faster_4_stages.prototxt") caffemodel = os.path.join( modelpath, "pose/mpi/pose_iter_160000.caffemodel") mpi_model = cv2.dnn.readNetFromCaffe(prototxt, caffemodel) return mpi_model def general_coco_model(self, modelpath): self.points_name = { "Nose": 0, "Neck": 1, "RShoulder": 2, "RElbow": 3, "RWrist": 4, "LShoulder": 5, "LElbow": 6, "LWrist": 7, "RHip": 8, "RKnee": 9, "RAnkle": 10, "LHip": 11, "LKnee": 12, "LAnkle": 13, "REye": 14, "LEye": 15, "REar": 16, "LEar": 17, "Background": 18} self.num_points = 18 self.point_pairs = [[1, 0], [1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [1, 8], [8, 9], [9, 10], [1, 11], [11, 12], [12, 13], [0, 14], [0, 15], [14, 16], [15, 17]] prototxt = os.path.join( modelpath, "pose/coco/pose_deploy_linevec.prototxt") caffemodel = os.path.join( modelpath, "pose/coco/pose_iter_440000.caffemodel") coco_model = cv2.dnn.readNetFromCaffe(prototxt, caffemodel) return coco_model def general_body25_model(self, modelpath): self.num_points = 25 self.point_pairs = [[1, 0], [1, 2], [1, 5], [2, 3], [3, 4], [5, 6], [6, 7], [0, 15], [15, 17], [0, 16], [16, 18], [1, 8], [8, 9], [9, 10], [10, 11], [11, 22], [22, 23], [11, 24], [8, 12], [12, 13], [13, 14], [14, 19], [19, 20], [14, 21]] prototxt = os.path.join( modelpath, "pose/body_25/pose_deploy.prototxt") caffemodel = os.path.join( modelpath, "pose/body_25/pose_iter_584000.caffemodel") coco_model = cv2.dnn.readNetFromCaffe(prototxt, caffemodel) return coco_model def predict(self, imgfile): img_cv2 = cv2.imread(imgfile) img_height, img_width, _ = img_cv2.shape inpBlob = cv2.dnn.blobFromImage(img_cv2, 1.0 / 255, (self.inWidth, self.inHeight), (0, 0, 0), swapRB=False, crop=False) self.pose_net.setInput(inpBlob) self.pose_net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) self.pose_net.setPreferableTarget(cv2.dnn.DNN_TARGET_OPENCL) output = self.pose_net.forward() H = output.shape[2] W = output.shape[3] print(output.shape) # vis heatmaps self.vis_heatmaps(img_file, output) # points = [] for idx in range(self.num_points): probMap = output[0, idx, :, :] # confidence map. # Find global maxima of the probMap. minVal, prob, minLoc, point = cv2.minMaxLoc(probMap) # Scale the point to fit on the original image x = (img_width * point[0]) / W y = (img_height * point[1]) / H if prob > self.threshold: points.append((int(x), int(y))) else: points.append(None) return points def vis_heatmaps(self, imgfile, net_outputs): img_cv2 = cv2.imread(imgfile) plt.figure(figsize=[10, 10]) for pdx in range(self.num_points): probMap = net_outputs[0, pdx, :, :] probMap = cv2.resize( probMap, (img_cv2.shape[1], img_cv2.shape[0]) ) plt.subplot(5, 5, pdx+1) plt.imshow(cv2.cvtColor(img_cv2, cv2.COLOR_BGR2RGB)) plt.imshow(probMap, alpha=0.6) plt.colorbar() plt.axis("off") plt.show() def vis_pose(self, imgfile, points): img_cv2 = cv2.imread(imgfile) img_cv2_copy = np.copy(img_cv2) for idx in range(len(points)): if points[idx]: cv2.circle(img_cv2_copy, points[idx], 8, (0, 255, 255), thickness=-1, lineType=cv2.FILLED) cv2.putText(img_cv2_copy, "{}".format(idx), points[idx], cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, lineType=cv2.LINE_AA) # Draw Skeleton for pair in self.point_pairs: partA = pair[0] partB = pair[1] if points[partA] and points[partB]: cv2.line(img_cv2, points[partA], points[partB], (0, 255, 255), 3) cv2.circle(img_cv2, points[partA], 8, (0, 0, 255), thickness=-1, lineType=cv2.FILLED) plt.figure(figsize=[10, 10]) plt.subplot(1, 2, 1) plt.imshow(cv2.cvtColor(img_cv2, cv2.COLOR_BGR2RGB)) plt.axis("off") plt.subplot(1, 2, 2) plt.imshow(cv2.cvtColor(img_cv2_copy, cv2.COLOR_BGR2RGB)) plt.axis("off") plt.show() if __name__ == '__main__': print("[INFO]Pose estimation.") img_file = "test.jpg" # start = time.time() modelpath = "/path/to/openpose_models/" pose_model = general_pose_model(modelpath, mode="BODY25") print("[INFO]Model loads time: ", time.time() - start) start = time.time() res_points = pose_model.predict(img_file) print("[INFO]Model predicts time: ", time.time() - start) pose_model.vis_pose(img_file, res_points)输出如：参考：[1] - Github - spmallick/learnopencv/OpenPose/[2] - Deep Learning based Human Pose Estimation using OpenCV ( C++ / Python ) - 2018.05.29
OpenCV 4.X 版本集成了很多直接利用 DNN 模块的 Python API 接口.安装：sudo pip install opencv-python使用：from cv2 import dnn# Variables with simple values DNN_BACKEND_DEFAULT = 0 DNN_BACKEND_HALIDE = 1 DNN_BACKEND_INFERENCE_ENGINE = 2 DNN_BACKEND_OPENCV = 3 DNN_BACKEND_VKCOM = 4 DNN_TARGET_CPU = 0 DNN_TARGET_MYRIAD = 3 DNN_TARGET_OPENCL = 1 DNN_TARGET_OPENCL_FP16 = 2 DNN_TARGET_VULKAN = 4 __loader__ = None __spec__ = None1. dnn.blobFromImage定义：def blobFromImage(image, scalefactor=None, size=None, mean=None, swapRB=None, crop=None, ddepth=None): pass作用：根据输入图像，创建 NCHW 次序的 4-dim blobs.参数：[1] - image: cv2.imread 读取的图片数据；[2] - scalefactor: 缩放像素值，如 [0, 255] - [0, 1].[3] - size: 输出图像的尺寸，如 (netInWidth, netInHeight).[4] - mean: 从各通道减均值. 如果输入 image 为 BGR 次序，且swapRB=True，则通道次序为 (mean-R, mean-G, mean-B).[5] - swapRB: 交换 3 通道图片的第一个和最后一个通道，如 BGR - RGB. [6] - crop: 图像尺寸 resize 后是否裁剪. 如果crop=True，则，输入图片的尺寸调整resize后，一个边对应与 size 的一个维度，而另一个边的值大于等于 size 的另一个维度；然后从 resize 后的图片中心进行 crop. 如果crop=False，则无需 crop，只需保持图片的长宽比.[7] - ddepth: 输出 blob 的 Depth. 可选: CV_32F 或 CV_8U.示例：import cv2 from cv2 import dnn import numpy as np import matplotlib.pyplot as plt img_cv2 = cv2.imread("test.jpg") print("[INFO]Image shape: ", img_cv2.shape) inWidth = 256 inHeight = 256 outBlob1 = cv2.dnn.blobFromImage(img_cv2, scalefactor=1.0 / 255, size=(inWidth, inHeight), mean=(0, 0, 0), swapRB=False, crop=False) print("[INFO]outBlob1 shape: ", outBlob1.shape) outimg1 = np.transpose(outBlob1[0], (1, 2, 0)) outBlob2 = cv2.dnn.blobFromImage(img_cv2, scalefactor=1.0 / 255, size=(inWidth, inHeight), mean=(0, 0, 0), swapRB=False, crop=True) print("[INFO]outBlob2 shape: ", outBlob2.shape) outimg2 = np.transpose(outBlob2[0], (1, 2, 0)) plt.figure(figsize=[10, 10]) plt.subplot(1, 3, 1) plt.title('Input image', fontsize=16) plt.imshow(cv2.cvtColor(img_cv2, cv2.COLOR_BGR2RGB)) plt.axis("off") plt.subplot(1, 3, 2) plt.title('Output image - no crop', fontsize=16) plt.imshow(cv2.cvtColor(outimg1, cv2.COLOR_BGR2RGB)) plt.axis("off") plt.subplot(1, 3, 3) plt.title('Output image - crop', fontsize=16) plt.imshow(cv2.cvtColor(outimg2, cv2.COLOR_BGR2RGB)) plt.axis("off") plt.show()2. dnn.blobFromImages定义：def blobFromImages(images, scalefactor=None, size=None, mean=None, swapRB=None, crop=None, ddepth=None): pass作用：批量处理图片，创建 4-dim blobs. 其它参数类似于 dnn.blobFromImage.3. dnn.Net_readFromModelOptimizer定义：def Net_readFromModelOptimizer(xml, bin): pass作用：从 Intel's Model Optimizer intermediate representation 创建网络.参数：[1] - xml: XML 网络拓扑结果的配置文件.[2] - bin: 训练权重值的二值文件.4. dnn.NMSBoxes定义：def NMSBoxes(bboxes, scores, score_threshold, nms_threshold, eta=None, top_k=None): pass作用：根据给定的 boxes 和对应的 scores 进行 NMS 处理.参数：[1] - boxes: 待处理的边界框 bounding boxes.[2] - scores: 对于于待处理边界框的 scores.[3] - score_threshold: 用于过滤 boxes 的 score 阈值.[4] - nms_threshold: NMS 用到的阈值.[5] - indices: NMS 处理后所保留的边界框的索引值.[6] - eta: 自适应阈值公式中的相关系数：$nms\_threshold_{i+1} = eta \cdot nms\_threshold_i$[7] - top_k: 如果 top_k>0，则保留最多 top_k 个边界框索引值.5. dnn.NMSBoxesRotated定义：def NMSBoxesRotated(bboxes, scores, score_threshold, nms_threshold, eta=None, top_k=None): pass6. dnn.readNet定义：def readNet(model, config=None, framework=None): pass作用：从支持的格式中加载深度学习网络和模型参数.参数：[1] - model: 训练的权重参数的模型二值文件，支持的格式有：*.caffemodel(Caffe)、*.pb(TensorFlow)、*.t7 或 *.net(Torch)、 *.weights(Darknet)、*.bin(DLDT).[2] - config: 包含网络配置的文本文件，支持的格式有：*.prototxt (Caffe)、*.pbtxt (TensorFlow)、*.cfg (Darknet)、*.xml (DLDT).[3] - framework: 所支持格式的框架名.该函数自动检测训练模型所采用的深度框架，然后调用 readNetFromCaffe、readNetFromTensorflow、readNetFromTorch 或 readNetFromDarknet 中的某个函数.7. dnn.readNetFromCaffe定义：def readNetFromCaffe(prototxt, caffeModel=None): pass作用：加载采用 Caffe 的配置网络和训练的权重参数.7. dnn.readNetFromDarknet定义：def readNetFromDarknet(cfgFile, darknetModel=None): pass作用：加载采用 Darknet 的配置网络和训练的权重参数.8. dnn.readNetFromModelOptimizer定义：def readNetFromModelOptimizer(xml, bin): pass作用：加载采用Intel's Model Optimizer intermediate representation 的配置网络和训练的权重参数.9. dnn.readNetFromONNX定义：def readNetFromONNX(onnxFile): pass作用：加载 .onnx 模型网络配置参数和权重参数.10. dnn.readNetFromTensorflow定义：def readNetFromTensorflow(model, config=None): pass作用：加载采用 Tensorflow 的配置网络和训练的权重参数.[1] - model: .pb 文件.[2] - config: .pbtxt 文件.11. dnn.readNetFromTorch定义：def readNetFromTorch(model, isBinary=None): pass作用：加载采用 Torch 的配置网络和训练的权重参数.[1] - model: 采用 torch.save() 函数保存的文件.所支持的 Torch nn.Module 网络层有：- nn.Sequential - nn.Parallel - nn.Concat - nn.Linear - nn.SpatialConvolution - nn.SpatialMaxPooling, nn.SpatialAveragePooling - nn.ReLU, nn.TanH, nn.Sigmoid - nn.Reshape - nn.SoftMax, nn.LogSoftMax12. dnn.readTensorFromONNX定义：def readTensorFromONNX(path): pass作用：从 .pb 文件创建 blob.[1] - path: 包含 input tensor 的.pb 文件.13. dnn.resetMyriadDevice定义：def resetMyriadDevice(): """ resetMyriadDevice() -> None . @brief Release a Myriad device is binded by OpenCV. . * . * Single Myriad device cannot be shared across multiple processes which uses . * Inference Engine's Myriad plugin. """ pass14. dnn.shrinkCaffeModel定义：def shrinkCaffeModel(src, dst, layersTypes=None): pass作用：将 Caffe 网络的所有权重转换为半精度浮点数值(half precision floating point).参数：[1] - src: Caffe 网路的原始单精度浮点数值权重模型文件(后缀一般为.caffemodel).[2] - dst: 转换后的权重文件.[3] - layersTypes: 待转换参数的网络层类型，默认是只转换卷积层和全连接层的权重参数.15. dnn.writeTextGraph定义：def writeTextGraph(model, output): #note: To reduce output file size, trained weights are not included. pass作用：将以 protocol buffer 格式的二值网络，创建为文本表示(Create a text representation for a binary network stored in protocol buffer format.)参数：[1] - model: 二值网络(binary network)的路径.[2] - output: 创建的输出文件路径.
Ubuntu16.04 中，OpenCV 多版本的升级与安装.1. OpenCV 编译安装1.1. 查看 Ubuntu 当前的 OpenCV 版本pkg-config --modversion opencv # 2.4.9.1在 /usr/bin 和 /usr/include路径里包含了默认 OpenCV 版本的相关文件.1.2. 依赖库安装[1] - 官方必须依赖库sudo apt-get update sudo apt-get install cmake git sudo apt-get install build-essential \ libgtk2.0-dev \ pkg-config \ libavcodec-dev \ libavformat-dev [2] - 官方建议依赖库sudo apt-get install python-dev \ libtbb2 \ libtbb-dev \ libjpeg-dev \ libpng-dev \ libtiff-dev \ libjasper-dev \ libdc1394-22-dev[3] - OPENGL 支持依赖库sudo apt-get install freeglut3-dev \ mesa-common-dev \ libgtkglext1 \ libgtkglext1-dev[4] - 视频解码支持依赖库sudo apt-get install checkinstall \ yasm \ libgstreamer0.10-dev \ libgstreamer-plugins-base0.10-dev \ libv4l-dev \ libtbb-dev \ libqt4-dev \ libgtk2.0-dev \ libmp3lame-dev \ libtheora-dev \ libvorbis-dev \ libxvidcore-dev \ x264 \ v4l-utils[5] - 其它可能依赖项sudo apt-get install libgphoto2-dev \ libavresample-dev \ liblapacke-dev \ gtk+-3.0 \ libgtk-3-dev \ libeigen3-dev \ tesseract-ocr \ liblept5 \ leptonica-progs \ libleptonica-dev 1.3. OpenCV 源码下载下载路径：https://github.com/opencv/opencv/releases1.4. OpenCV 3.4 编译安装假设路径为 ROOT_DIR = /opt/opencv3.4/cd $ROOT_DIR mkdir build cd build cmake -D CMAKE_BUILD_TYPE=RELEASE \ -D CMAKE_INSTALL_PREFIX=/usr/local/opencv34 .. # cmake -D WITH_TBB=ON # -D BUILD_NEW_PYTHON_SUPPORT=ON \ # -D WITH_V4L=ON \ # -D INSTALL_C_EXAMPLES=ON \ # -D INSTALL_PYTHON_EXAMPLES=ON \ # -D BUILD_EXAMPLES=ON \ # -D WITH_QT=ON \ # -D WITH_OPENGL=ON .. make -j4 sudo make install在编译安装时，ippicv 库的下载文件 - ippicv_2019_lnx_intel64_general_20180723.tgz 速度特别慢.因此，采用离线下载文件，再进行编译.ippicv_2019_lnx_intel64_general_20180723.tgz 文件下载：百度网盘链接: https://pan.baidu.com/s/1mVTSdn6hTgo8P3ASKauyDg 提取码: hiux假设 ippicv_2019_lnx_intel64_general_20180723.tgz放置于 ROOT_DIR/3rdparty/ippicv/ 路径，并修改 ROOT_DIR/3rdparty/ippicv/ippicv.cmake 文件(47 行)：修改前：41 set(THE_ROOT "${OpenCV_BINARY_DIR}/3rdparty/ippicv") 42 ocv_download(FILENAME ${OPENCV_ICV_NAME} 43 HASH ${OPENCV_ICV_HASH} 44 URL 45 "${OPENCV_IPPICV_URL}" 46 "$ENV{OPENCV_IPPICV_URL}" 47 "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${IPPICV_COMMIT}/ippicv/" 48 DESTINATION_DIR "${THE_ROOT}" 49 ID IPPICV 50 STATUS res 51 UNPACK RELATIVE_URL)修改后：41 set(THE_ROOT "${OpenCV_BINARY_DIR}/3rdparty/ippicv") 42 ocv_download(FILENAME ${OPENCV_ICV_NAME} 43 HASH ${OPENCV_ICV_HASH} 44 URL 45 "${OPENCV_IPPICV_URL}" 46 "$ENV{OPENCV_IPPICV_URL}" 47 "file: ROOT_DIR/3rdparty/ippicv/ippicv_2019_lnx_intel64_general_20180723.tgz" 48 DESTINATION_DIR "${THE_ROOT}" 49 ID IPPICV 50 STATUS res 51 UNPACK RELATIVE_URL)修改后，下载 ippicv文件时，会自动从本地文件加载.1.5. OpenCV 4.0.1 (最新版本) 编译安装假设路径为 ROOT_DIR = /opt/opencv-4.0.1/cd $ROOT_DIR mkdir build cd build cmake -D CMAKE_BUILD_TYPE=RELEASE \ -D BUILD_NEW_PYTHON_SUPPORT=ON \ -D CMAKE_INSTALL_PREFIX=/usr/local/opencv-4.0.1 \ -D OPENCV_GENERATE_PKGCONFIG=ON .. #cmake -D CMAKE_BUILD_TYPE=RELEASE \ # -D CMAKE_INSTALL_PREFIX=/usr/local/opencv-4.0.1 \ # -D WITH_CUDA=ON \ # -D OPENCV_EXTRA_MODULES_PATH=/opt/opencv_contrib/modules # -D WITH_CUBLAS=1 \ # -D ENABLE_FAST_MATH=1 \ # -D CUDA_FAST_MATH=1 \ # -D WITH_CUFFT=ON \ # -D WITH_NVCUVID=ON \ # -D WITH_V4L=ON \ # -D WITH_LIBV4L=ON \ # -D WITH_OPENGL=ON \ # -D WITH_FFMPEG=ON \ # -D BUILD_TIFF=ON \ # -D WITH_LAPACK=OFF.. # -D INSTALL_C_EXAMPLES=ON \ # -D BUILD_EXAMPLES=ON .. make sudo make install1.5.1 OpenCV 4.0.1 CUDA 编译问题[1] - -DWITH_CUDA=ON 需要设置 -DOPENCV_EXTRA_MODULES_PATH=<opencv_contrib>/modules[2] - 如果没有contrib, make 的时候会出现错误：gpu_mat.cu:47:2: error: #error "opencv_cudev is required".Github - opencv_contrib[3] - 在 opencv_contrib/modules/face 中会要下载 face_landmark_model.dat 文件(69M)，也会遇到下载慢的问题，可以采用类似于 ippicv 库的处理方式.face_landmark_model.mat 下载路径：https://github.com/opencv/opencv_3rdparty/tree/contrib_face_alignment_20170818在opencv_contrib/modules/face/CMakeLists.txt 文件：set(__commit_hash "8afa57abc8229d611c4937165d20e2a2d9fc5a12") set(__file_hash "7505c44ca4eb54b4ab1e4777cb96ac05") ocv_download( FILENAME face_landmark_model.dat HASH ${__file_hash} URL "${OPENCV_FACE_ALIGNMENT_URL}" "$ENV{OPENCV_FACE_ALIGNMENT_URL}" "https://raw.githubusercontent.com/opencv/opencv_3rdparty/${__commit_hash}/" DESTINATION_DIR "${CMAKE_BINARY_DIR}/${OPENCV_TEST_DATA_INSTALL_PATH}/cv/face/" ID "data" RELATIVE_URL STATUS res ) if(NOT res) message(WARNING "Face: Can't get model file for face alignment.") endif()将离线下载的 face_landmark_model.dat 放到 /opt/opencv-4.0.1/build/share/opencv4/testdata/cv/face/ 路径中.1.6. 环境变量设置配置 OpenCV 编译环境：编辑或新建 /etc/ld.so.conf.d/opencv.conf:sudo gedit /etc/ld.so.conf.d/opencv.conf文件尾部添加如下内容：/usr/local/opencv-4.0.1/lib生效配置文件：sudo ldconfigsudo gedit ~/.bashrc在该文件尾部添加如下行内容：export PKG_CONFIG_PATH=/usr/local/opencv-4.0.1/lib/pkgconfig:$PKG_CONFIG_PATH export LD_LIBRARY_PATH=/usr/local/opencv-4.0.1/lib:$LD_LIBRARY_PATH环境变量生效：source ~/.bashrc查看opencv版本及库信息：pkg-config --modversion opencv4 # 4.0.1 pkg-config --libs opencv4 pkg-config --cflags opencv41.7. OpenCV 的 cmake 使用使用 cmake 工具时，在 CMakeLists.txt 文件中添加一下内容：set(OpenCV_DIR /usr/local/opencv-4.0.1/share/OpenCV) find_package(OpenCV 4.0.1 REQUIRED) include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS} /usr/local/opencv-4.0.1/include)1.8. Python 接口sudo ln -s \ /usr/local/opencv-4.0.1/lib/python2.7/dist-packages/cv2/python-2.7/cv2.so \ /usr/local/lib/python2.7/dist-packages/cv2/cv2.so sudo ln -s \ /usr/local/opencv-4.0.1/lib/python3.5/dist-packages/cv2.cpython-35m-x86_64-linux-gnu.so \ /usr/local/lib/python3.5/dist-packages/cv2/cv2.cpython-35m-x86_64-linux-gnu.so2. OpenCV 测试以 opencv-4.0.1 为例.假设项目路径为：/path/to/opencv_demo/:2.1. 例 - 显示图片新建 opencv_test.cpp 文件：#include <iostream> #include <opencv2/opencv.hpp> using namespace cv; using namespace std; int main(int argc, char** argv) { if ( argc != 2 ) { cout << "Need to load an image ..." << endl; return -1; } Mat image; image = imread( argv[1], 1 ); if ( image.empty() ) { cout<<"No image data!"<< endl; return -1; } namedWindow("Display Image", WINDOW_AUTOSIZE ); imshow("Display Image", image); waitKey(0); return 0; }新建 CMakeLists.txt 文件：cmake_minimum_required(VERSION 2.8) project( DisplayImage ) set(OpenCV_DIR /usr/local/opencv-4.0.1/lib/cmake/opencv4) find_package(OpenCV REQUIRED ) add_executable( opencv_test opencv_test.cpp ) target_link_libraries( opencv_test ${OpenCV_LIBS} )执行：cmake . -DCMAKE_CXX_FLAGS="-std=c++11" # opencv4.0 需要 c++ 11 支持 make ./opencv_test test.jpg显示图片.2.2. 例 - SURF 特征点检测新建 surf_test.cpp 文件：#include <iostream> #include <vector> #include <opencv2/xfeatures2d.hpp> #include <opencv2/highgui.hpp> using namespace cv; using namespace std; int main(int argc, char** argv) { Mat img1 = imread(argv[1], 1); Mat img2 = imread(argv[2], 1); if ((img1.data == NULL)||(img2.data == NULL)) { cout << "No images exist" << endl; return -1; } Ptr<Feature2D> surf = xfeatures2d::SURF::create(1000); vector<KeyPoint> keypoints_1, keypoints_2; Mat descriptors_1, descriptors_2; surf->detectAndCompute(img1, Mat(), keypoints_1, descriptors_1 ); surf->detectAndCompute(img2, Mat(), keypoints_2, descriptors_2 ); drawKeypoints(img1, keypoints_1, img1); drawKeypoints(img2, keypoints_2, img2); namedWindow("img1",0); resizeWindow("img1",500,500); imshow("img1", img1); namedWindow("img2",0); resizeWindow("img2",500,500); imshow("img2", img2); FlannBasedMatcher matcher; std::vector< DMatch > matches; matcher.match( descriptors_1, descriptors_2, matches ); double max_dist = 0; double min_dist = 100; for( int i = 0; i < descriptors_1.rows; i++ ) { double dist = matches[i].distance; if( dist < min_dist ) min_dist = dist; if( dist > max_dist ) max_dist = dist; } printf("-- Max dist : %f \n", max_dist ); printf("-- Min dist : %f \n", min_dist ); std::vector< DMatch > good_matches; for( int i = 0; i < descriptors_1.rows; i++ ) { if( matches[i].distance <= max(2*min_dist, 0.02) ) { good_matches.push_back( matches[i]); } } Mat img_matches; drawMatches( img1, keypoints_1, img2, keypoints_2, good_matches, img_matches, Scalar::all(-1), Scalar::all(-1), vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS ); namedWindow("Good Matches",0); resizeWindow("Good Matches",800,800); imshow( "Good Matches", img_matches ); for( int i = 0; i < (int)good_matches.size(); i++ ) { printf( "-- Good Match [%d] Keypoint 1: %d -- Keypoint 2: %d \n", i, good_matches[i].queryIdx, good_matches[i].trainIdx ); } waitKey(0); return 0; }新建 CMakeLists.txt 文件：set(CMAKE_CXX_FLAGS "-std=c++11") cmake_minimum_required(VERSION 2.8) project( SURF_test ) set(OpenCV_DIR /usr/local/opencv-4.0.1/lib/cmake/opencv4) find_package( OpenCV REQUIRED ) add_executable( SURF_test surf_test.cpp ) target_link_libraries( SURF_test ${OpenCV_LIBS} )执行：cmake . make ./surf_test c1.jpg c2.jpg