人脸识别，人脸检测，人脸关键点，人脸对比
在 StyleGAN2 中，可以采用基于 Dlib 库实现人脸关键点检测、人脸对齐.1. 人脸关键点检测import dlib #predictor_model_path='http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2' class LandmarksDetector: def __init__(self, predictor_model_path): """ :param predictor_model_path: path to shape_predictor_68_face_landmarks.dat file """ self.detector = dlib.get_frontal_face_detector() # cnn_face_detection_model_v1 also can be used self.shape_predictor = dlib.shape_predictor(predictor_model_path) def get_landmarks(self, image): img = dlib.load_rgb_image(image) dets = self.detector(img, 1) for detection in dets: try: face_landmarks = [(item.x, item.y) for item in self.shape_predictor(img, detection).parts()] yield face_landmarks except: print("Exception in get_landmarks()!")2. 人脸对齐import numpy as np import scipy.ndimage import os import PIL.Image def image_align(src_file, dst_file, face_landmarks, output_size=1024, transform_size=4096, enable_padding=True, x_scale=1, y_scale=1, em_scale=0.1, alpha=False): # Align function from FFHQ dataset pre-processing step # https://github.com/NVlabs/ffhq-dataset/blob/master/download_ffhq.py lm = np.array(face_landmarks) lm_chin = lm[0 : 17] # left-right lm_eyebrow_left = lm[17 : 22] # left-right lm_eyebrow_right = lm[22 : 27] # left-right lm_nose = lm[27 : 31] # top-down lm_nostrils = lm[31 : 36] # top-down lm_eye_left = lm[36 : 42] # left-clockwise lm_eye_right = lm[42 : 48] # left-clockwise lm_mouth_outer = lm[48 : 60] # left-clockwise lm_mouth_inner = lm[60 : 68] # left-clockwise # Calculate auxiliary vectors. eye_left = np.mean(lm_eye_left, axis=0) eye_right = np.mean(lm_eye_right, axis=0) eye_avg = (eye_left + eye_right) * 0.5 eye_to_eye = eye_right - eye_left mouth_left = lm_mouth_outer[0] mouth_right = lm_mouth_outer[6] mouth_avg = (mouth_left + mouth_right) * 0.5 eye_to_mouth = mouth_avg - eye_avg # Choose oriented crop rectangle. x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1] x /= np.hypot(*x) x *= max(np.hypot(*eye_to_eye) * 2.0, np.hypot(*eye_to_mouth) * 1.8) x *= x_scale y = np.flipud(x) * [-y_scale, y_scale] c = eye_avg + eye_to_mouth * em_scale quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y]) qsize = np.hypot(*x) * 2 # Load in-the-wild image. if not os.path.isfile(src_file): print('\nCannot find source image. Please run "--wilds" before "--align".') return img = PIL.Image.open(src_file).convert('RGBA').convert('RGB') # Shrink. shrink = int(np.floor(qsize / output_size * 0.5)) if shrink > 1: rsize = (int(np.rint(float(img.size[0]) / shrink)), int(np.rint(float(img.size[1]) / shrink))) img = img.resize(rsize, PIL.Image.ANTIALIAS) quad /= shrink qsize /= shrink # Crop. border = max(int(np.rint(qsize * 0.1)), 3) crop = (int(np.floor(min(quad[:,0]))), int(np.floor(min(quad[:,1]))), int(np.ceil(max(quad[:,0]))), int(np.ceil(max(quad[:,1])))) crop = (max(crop[0] - border, 0), max(crop[1] - border, 0), min(crop[2] + border, img.size[0]), min(crop[3] + border, img.size[1])) if crop[2] - crop[0] < img.size[0] or crop[3] - crop[1] < img.size[1]: img = img.crop(crop) quad -= crop[0:2] # Pad. pad = (int(np.floor(min(quad[:,0]))), int(np.floor(min(quad[:,1]))), int(np.ceil(max(quad[:,0]))), int(np.ceil(max(quad[:,1])))) pad = (max(-pad[0] + border, 0), max(-pad[1] + border, 0), max(pad[2] - img.size[0] + border, 0), max(pad[3] - img.size[1] + border, 0)) if enable_padding and max(pad) > border - 4: pad = np.maximum(pad, int(np.rint(qsize * 0.3))) img = np.pad(np.float32(img), ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect') h, w, _ = img.shape y, x, _ = np.ogrid[:h, :w, :1] mask = np.maximum(1.0 - np.minimum(np.float32(x) / pad[0], np.float32(w-1-x) / pad[2]), 1.0 - np.minimum(np.float32(y) / pad[1], np.float32(h-1-y) / pad[3])) blur = qsize * 0.02 img += (scipy.ndimage.gaussian_filter(img, [blur, blur, 0]) - img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0) img += (np.median(img, axis=(0,1)) - img) * np.clip(mask, 0.0, 1.0) img = np.uint8(np.clip(np.rint(img), 0, 255)) if alpha: mask = 1-np.clip(3.0 * mask, 0.0, 1.0) mask = np.uint8(np.clip(np.rint(mask*255), 0, 255)) img = np.concatenate((img, mask), axis=2) img = PIL.Image.fromarray(img, 'RGBA') else: img = PIL.Image.fromarray(img, 'RGB') quad += pad[:2] # Transform. img = img.transform((transform_size, transform_size), PIL.Image.QUAD, (quad + 0.5).flatten(), PIL.Image.BILINEAR) if output_size < transform_size: img = img.resize((output_size, output_size), PIL.Image.ANTIALIAS) # Save aligned image. img.save(dst_file, 'PNG')3. 人脸检测及对齐# from https://github.com/rolux import os import sys import bz2 from face_alignment import image_align from landmarks_detector import LandmarksDetector def unpack_bz2(src_path): data = bz2.BZ2File(src_path).read() dst_path = src_path[:-4] with open(dst_path, 'wb') as fp: fp.write(data) return dst_path if __name__ == "__main__": """ Extracts and aligns all faces from images using DLib and a function from original FFHQ dataset preparation step python align_images.py /raw_images /aligned_images """ #'http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2' landmarks_model_path = unpack_bz2('./shape_predictor_68_face_landmarks.dat.bz2') RAW_IMAGES_DIR = sys.argv[1] ALIGNED_IMAGES_DIR = sys.argv[2] landmarks_detector = LandmarksDetector(landmarks_model_path) for img_name in [x for x in os.listdir(RAW_IMAGES_DIR) if x[0] not in '._']: raw_img_path = os.path.join(RAW_IMAGES_DIR, img_name) for i, face_landmarks in enumerate(landmarks_detector.get_landmarks(raw_img_path), start=1): face_img_name = '%s_%02d.png' % (os.path.splitext(img_name)[0], i) aligned_face_path = os.path.join(ALIGNED_IMAGES_DIR, face_img_name) os.makedirs(ALIGNED_IMAGES_DIR, exist_ok=True) image_align(raw_img_path, aligned_face_path, face_landmarks)使用：python align_images.py ./unaligned_images ./aligned_images
基于OpenCV和dlib的人脸性别及年龄检测的实现
采用 OpenCV DNN 模块实现的人脸性别及年龄检测，整个项目比较简单、清晰明了，过程主要包括：[1] - 检测图片中的人脸框(如，采用 dlib 库).[2] - 检测人脸的性别和年龄.[3] - 检测结果可视化.1. 相关模型所采用的模型分别如下：[1] - 人脸框检测模型这里采用了 dlib 的 CNN 人脸框检测模型：mmod_human_face_detector.dat.bz2或者，OpenCV DNN 模块自带的残差网络的人脸检测模型：opencv_face_detector.pbtxtopencv_face_detector_uint8.pb[2] - 性别检测模型(Caffe)：gender_deploy.prototxtgender_net.caffemodel性别共两类：Male，Female[3] - 年龄预测模型(Caffe)：age_deploy.prototxtage_net.caffemodel年龄共八类：(0-2), (4-6), (8-12), (15-20), (25-32), (38-43), (48-53), (60-100)2. 具体实现#!--*-- coding:utf-8 --*-- import os import cv2 import time import matplotlib.pyplot as plt import dlib class AgeGenderDetector(): def __init__(self): # Load model and networks # Age self.ageList = ['(0-2)', '(4-6)', '(8-12)', '(15-20)', '(25-32)', '(38-43)', '(48-53)', '(60-100)'] ageProto = "age_deploy.prototxt" ageModel = "age_net.caffemodel" self.ageNet = cv2.dnn.readNet(ageModel, ageProto) # Gender self.genderList = ['Male', 'Female'] genderProto = "gender_deploy.prototxt" genderModel = "gender_net.caffemodel" self.genderNet = cv2.dnn.readNet(genderModel, genderProto) # Face detection - OpenCV DNN faceProto = "opencv_face_detector.pbtxt" faceModel = "opencv_face_detector_uint8.pb" self.faceNet = cv2.dnn.readNet(faceModel, faceProto) # Face detection - dlib dlib_cnn_face_model = "mmod_human_face_detector.dat" self.dlib_cnn_face_detector = dlib.cnn_face_detection_model_v1(dlib_cnn_face_model) self.mean_values = (78.4263377603, 87.7689143744, 114.895847746) self.padding = 20 def dlib_cnn_face_detect(self, img): detections = self.dlib_cnn_face_detector(img, 1) bboxes = [] if len(detections) > 0: for i, d in enumerate(detections): x1 = d.rect.left() y1 = d.rect.top() x2 = d.rect.right() y2 = d.rect.bottom() bboxes.append([x1, y1, x2, y2]) # return bboxes def opencv_dnn_face_detect(self, img_cv2, conf_threshold=0.7): height, width, _ = img_cv2.shape blob = cv2.dnn.blobFromImage(img_cv2, 1.0, (300, 300), [104, 117, 123], True, False) self.faceNet.setInput(blob) detections = self.faceNet.forward() bboxes = [] for i in range(detections.shape[2]): confidence = detections[0, 0, i, 2] if confidence > conf_threshold: x1 = int(detections[0, 0, i, 3] * width) y1 = int(detections[0, 0, i, 4] * height) x2 = int(detections[0, 0, i, 5] * width) y2 = int(detections[0, 0, i, 6] * height) bboxes.append([x1, y1, x2, y2]) # return bboxes def detect(self, img_file): start = time.time() img_cv2 = cv2.imread(img_file) img_rgb = cv2.cvtColor(img_cv2, cv2.COLOR_BGR2RGB) # Face box detection # face_bboxes = self.opencv_dnn_face_detect(img_cv2) face_bboxes = self.dlib_cnn_face_detect(img_rgb) # RGB order if not face_bboxes: print("No face Detected, Checking next image") for bbox in face_bboxes: # print(bbox) # single face face = img_cv2[ max(0, bbox[1]-self.padding): min(bbox[3] + self.padding, img_cv2.shape[0]-1), max(0, bbox[0]-self.padding): min(bbox[2] + self.padding, img_cv2.shape[1]-1)] # Gender blob = cv2.dnn.blobFromImage(face, 1.0, (227, 227), self.mean_values, swapRB=False) self.genderNet.setInput(blob) genderPreds = self.genderNet.forward() gender = self.genderList[genderPreds[0].argmax()] # print("Gender Output : {}".format(genderPreds)) print("Gender : {}, conf = {:.3f}".format( gender, genderPreds[0].max())) # Age self.ageNet.setInput(blob) agePreds = self.ageNet.forward() age = self.ageList[agePreds[0].argmax()] print("Age Output : {}".format(agePreds)) print("Age : {}, conf = {:.3f}".format(age, agePreds[0].max())) # vis label = "{}, {}".format(gender, age) cv2.rectangle(img_cv2, (bbox[0], bbox[1]), (bbox[2], bbox[3]), (0, 255, 0), int(round(img_cv2.shape[0]/150)), 8) cv2.putText(img_cv2, label, (bbox[0], bbox[1]-10), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 2, cv2.LINE_AA) # cv2.imwrite("age-gender-out-{}".format(args.input),frameFace) print("time : {:.3f}".format(time.time() - start)) plt.figure(figsize=(10, 8)) plt.axis("off") plt.imshow(img_cv2[:,:,::-1]) plt.show() return True if __name__ == '__main__': print("[INFO] Starting AgeGender Demo...") img_path = "/path/to/images/" img_files = [os.path.join(img_path, img_name) for img_name in os.listdir(img_path)] age_gender_detector = AgeGenderDetector() for img_file in img_files: print("Processing file: {}".format(img_file)) age_gender_detector.detect(img_file) print("[INFO]Done.")输入如：Gender : Female, conf = 0.984 Age : (25-32), conf = 0.500 time : 0.474Gender : Female, conf = 0.999 Age : (25-32), conf = 0.408 time : 0.2113. 参考材料主要参考：[1] - Age and Gender Classification Using Convolutional Neural Network[2] - Github - spmallick/learnopencv/AgeGender [3] - 论文阅读 - 基于CNN的年龄和性别分类 - AIUAI
1. 基于 OpenCV 和 dlib 的实时人脸特征点检测原文：Detect eyes, nose, lips, and jaw with dlib, OpenCV, and Python - 2017.04.10原文：Real-time facial landmark detection with OpenCV, Python, and dlib - 2017.04.17作者：Adrian Rosebrock安装 imutils 和 dlib：pip install dlib pip install imutils #或 pip install --upgrade imutilsdlib 库检测的人脸特征点如图：以视频流人脸特征点检测为例 - video_facial_landmarks.py:from imutils.video import VideoStream from imutils import face_utils import datetime import argparse import imutils import time import dlib import cv2 # construct the argument parse and parse the arguments ap = argparse.ArgumentParser() ap.add_argument("-p", "--shape-predictor", required=True, help="path to facial landmark predictor") ap.add_argument("-r", "--picamera", type=int, default=-1, help="whether or not the Raspberry Pi camera should be used") args = vars(ap.parse_args()) # initialize dlib's face detector (HOG-based) and then create # the facial landmark predictor print("[INFO] loading facial landmark predictor...") detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(args["shape_predictor"]) # initialize the video stream and allow the cammera sensor to warmup print("[INFO] camera sensor warming up...") vs = VideoStream(usePiCamera=args["picamera"] > 0).start() time.sleep(2.0) # loop over the frames from the video stream while True: # grab the frame from the threaded video stream, resize it to # have a maximum width of 400 pixels, and convert it to # grayscale frame = vs.read() frame = imutils.resize(frame, width=400) gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # detect faces in the grayscale frame rects = detector(gray, 0) # import the necessary packages from imutils.video import VideoStream from imutils import face_utils import datetime import argparse import imutils import time import dlib import cv2 # construct the argument parse and parse the arguments ap = argparse.ArgumentParser() ap.add_argument("-p", "--shape-predictor", required=True, help="path to facial landmark predictor") ap.add_argument("-r", "--picamera", type=int, default=-1, help="whether or not the Raspberry Pi camera should be used") args = vars(ap.parse_args()) # initialize dlib's face detector (HOG-based) and then create # the facial landmark predictor print("[INFO] loading facial landmark predictor...") detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(args["shape_predictor"]) # initialize the video stream and allow the cammera sensor to warmup print("[INFO] camera sensor warming up...") vs = VideoStream(usePiCamera=args["picamera"] > 0).start() time.sleep(2.0) # loop over the frames from the video stream while True: # grab the frame from the threaded video stream, resize it to # have a maximum width of 400 pixels, and convert it to # grayscale frame = vs.read() frame = imutils.resize(frame, width=400) gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # detect faces in the grayscale frame rects = detector(gray, 0) # loop over the face detections for rect in rects: # determine the facial landmarks for the face region, then # convert the facial landmark (x, y)-coordinates to a NumPy # array shape = predictor(gray, rect) shape = face_utils.shape_to_np(shape) # loop over the (x, y)-coordinates for the facial landmarks # and draw them on the image for (x, y) in shape: cv2.circle(frame, (x, y), 1, (0, 0, 255), -1) # show the frame cv2.imshow("Frame", frame) key = cv2.waitKey(1) & 0xFF # if the `q` key was pressed, break from the loop if key == ord("q"): break # import the necessary packages from imutils.video import VideoStream from imutils import face_utils import datetime import argparse import imutils import time import dlib import cv2 # construct the argument parse and parse the arguments ap = argparse.ArgumentParser() ap.add_argument("-p", "--shape-predictor", required=True, help="path to facial landmark predictor") ap.add_argument("-r", "--picamera", type=int, default=-1, help="whether or not the Raspberry Pi camera should be used") args = vars(ap.parse_args()) # initialize dlib's face detector (HOG-based) and then create # the facial landmark predictor print("[INFO] loading facial landmark predictor...") detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(args["shape_predictor"]) # initialize the video stream and allow the cammera sensor to warmup print("[INFO] camera sensor warming up...") vs = VideoStream(usePiCamera=args["picamera"] > 0).start() time.sleep(2.0) # loop over the frames from the video stream while True: # grab the frame from the threaded video stream, resize it to # have a maximum width of 400 pixels, and convert it to # grayscale frame = vs.read() frame = imutils.resize(frame, width=400) gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # detect faces in the grayscale frame rects = detector(gray, 0) # loop over the face detections for rect in rects: # determine the facial landmarks for the face region, then # convert the facial landmark (x, y)-coordinates to a NumPy # array shape = predictor(gray, rect) shape = face_utils.shape_to_np(shape) # loop over the (x, y)-coordinates for the facial landmarks # and draw them on the image for (x, y) in shape: cv2.circle(frame, (x, y), 1, (0, 0, 255), -1) # show the frame cv2.imshow("Frame", frame) key = cv2.waitKey(1) & 0xFF # if the `q` key was pressed, break from the loop if key == ord("q"): break # do a bit of cleanup cv2.destroyAllWindows() vs.stop()运行：$ python video_facial_landmarks.py \ --shape-predictor shape_predictor_68_face_landmarks.dat如：2. 基于 OpenCV 和 dlib 的眨眼检测原文：Eye blink detection with OpenCV, Python, and dlib -2017.04.24图像识别的新思路：眼睛纵横比，玩转识别眨眼动作. 基于面部特征检测和计算视频流中的眨眼次数.眨眼次数检测器，将对眼睛长宽比(eye aspect ratio, EAR)进行计算，可参考 2016年的论文Real-Time Eye Blink Detection Using Facial Landmarks.传统计算眨眼次数的图像处理方法，主要包括：[1] - 眼睛定位(eye localiatioon)[2] - 设定寻找眼睛白色部分的阈值(Thresholding to find the whites of the eyes.)[3] - 判断眼睛白色区域消失的周期次数(消失一次，表示眨眼一次).计算眼睛的长宽比(EAR) 的方式是一种更简洁优雅的方式，其主要涉及到基于眼睛的面部特征点之家安的距离比例的简单计算. 这种眨眼检测的方法是快速、有效，且易于实现的.2.1. 眼睛长宽比(EAR) 的理解采用 OpenCV 和 dlib 可以检测到人脸特征点，定位到脸部的重要区域，如眼睛，眉毛，鼻子，嘴巴，耳朵等.对于眨眼检测而言，只关心于人脸的两只眼睛区域.每个眼睛表示为 6 个 (x, y) 坐标，从眼睛的左角开始(正对于人脸而言)，然后，顺时针到剩余部分:眼睛的 6 个面部特征点.基于此，可以发现眼睛特征点坐标的长和宽之间的关系.根据 2016 年的论文 Real-Time Eye Blink Detection using Facial Landmarks，能够得到反映眼睛特征点之间关系的数学公式，即眼睛长宽比(EAR):$$ EAR = \frac{||p_2 - p_6|| + ||p_3 - p_5||}{2||p_1 - p_4||} $$其中，$p_1, ..., p_6​$ 为面部特征点位置.该 EAR 公式是计算垂直眼睛标志之间的距离，分母是计算水平眼睛标志之间的距离，因为水平点只有一组，但垂直点有两组，所以分母进行了加权.EAR 公式有趣的地方是，EAR 在眼睛张开时大致是保持恒定不变的，但在眨眼时却会发生迅速降到零的变化.利用 EAR 公式，可以避免使用图像处理技术，只需要计算眼睛特征关键点的长宽比，就可以判断一个人是否眨眼.更清晰的说明，如图：图：(上左) 眼睛睁开时的特征点的可视化；(上右) 眼睛闭上时的特征点的可视化；(下) 一段时间内的眼睛长宽比情况. 明显下降的地方表示一次眨眼.2.2. 基于人脸关键点的眨眼检测 python 实现detect_blinks.py:from scipy.spatial import distance as dist from imutils.video import FileVideoStream from imutils.video import VideoStream from imutils import face_utils import numpy as np import argparse import imutils import time import dlib import cv2 # EAR 计算 def eye_aspect_ratio(eye): # compute the euclidean distances between the two sets of # vertical eye landmarks (x, y)-coordinates A = dist.euclidean(eye[1], eye[5]) B = dist.euclidean(eye[2], eye[4]) # compute the euclidean distance between the horizontal # eye landmark (x, y)-coordinates C = dist.euclidean(eye[0], eye[3]) # compute the eye aspect ratio ear = (A + B) / (2.0 * C) # return the eye aspect ratio return ear # construct the argument parse and parse the arguments ap = argparse.ArgumentParser() ap.add_argument("-p", "--shape-predictor", required=True, help="path to facial landmark predictor") ap.add_argument("-v", "--video", type=str, default="", help="path to input video file") args = vars(ap.parse_args()) # define two constants, one for the eye aspect ratio to indicate # blink and then a second constant for the number of consecutive # frames the eye must be below the threshold EYE_AR_THRESH = 0.3 EYE_AR_CONSEC_FRAMES = 3 # initialize the frame counters and the total number of blinks COUNTER = 0 TOTAL = 0 # initialize dlib's face detector (HOG-based) and then create # the facial landmark predictor print("[INFO] loading facial landmark predictor...") detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor(args["shape_predictor"]) # grab the indexes of the facial landmarks for the left and # right eye, respectively (lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"] (rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"] # start the video stream thread print("[INFO] starting video stream thread...") vs = FileVideoStream(args["video"]).start() fileStream = True # vs = VideoStream(src=0).start() # vs = VideoStream(usePiCamera=True).start() # fileStream = False time.sleep(1.0) # loop over frames from the video stream while True: # if this is a file video stream, then we need to check if # there any more frames left in the buffer to process if fileStream and not vs.more(): break # grab the frame from the threaded video file stream, resize # it, and convert it to grayscale # channels) frame = vs.read() frame = imutils.resize(frame, width=450) gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # detect faces in the grayscale frame rects = detector(gray, 0) # loop over the face detections for rect in rects: # determine the facial landmarks for the face region, then # convert the facial landmark (x, y)-coordinates to a NumPy # array shape = predictor(gray, rect) shape = face_utils.shape_to_np(shape) # extract the left and right eye coordinates, then use the # coordinates to compute the eye aspect ratio for both eyes leftEye = shape[lStart:lEnd] rightEye = shape[rStart:rEnd] leftEAR = eye_aspect_ratio(leftEye) rightEAR = eye_aspect_ratio(rightEye) # average the eye aspect ratio together for both eyes ear = (leftEAR + rightEAR) / 2.0 # compute the convex hull for the left and right eye, then # visualize each of the eyes leftEyeHull = cv2.convexHull(leftEye) rightEyeHull = cv2.convexHull(rightEye) cv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1) cv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1) # check to see if the eye aspect ratio is below the blink # threshold, and if so, increment the blink frame counter if ear < EYE_AR_THRESH: COUNTER += 1 # otherwise, the eye aspect ratio is not below the blink # threshold else: # if the eyes were closed for a sufficient number of # then increment the total number of blinks if COUNTER >= EYE_AR_CONSEC_FRAMES: TOTAL += 1 # reset the eye frame counter COUNTER = 0 # draw the total number of blinks on the frame along with # the computed eye aspect ratio for the frame cv2.putText(frame, "Blinks: {}".format(TOTAL), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) cv2.putText(frame, "EAR: {:.2f}".format(ear), (300, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) # show the frame cv2.imshow("Frame", frame) key = cv2.waitKey(1) & 0xFF # if the `q` key was pressed, break from the loop if key == ord("q"): break # do a bit of cleanup cv2.destroyAllWindows() vs.stop()运行：# 视频输入 python detect_blinks.py \ --shape-predictor shape_predictor_68_face_landmarks.dat \ --video blink_detection_demo.mp4 # 摄像头输入 python detect_blinks.py \ --shape-predictor shape_predictor_68_face_landmarks.dat2.3. 眨眼检测器改进这里仅关注眼睛长宽比作为量化指标，以判断人是否眨眼. 但，由于噪声、人脸特征点检测、以及视角的快速变化，采用简单的设定长宽比阈值的方法可能会出现 false-positive 检测结果，也就是检测到人眨眼了，但实际上并没有眨眼.眨眼检测器的改进，以更鲁棒，EAR 论文作者建议：[1] - 首先，同时计算第 N 帧和第 N-6 和 N+6 帧的眼睛长宽比，得到 13 维特征向量；[2] - 然后，将特征向量采用 SVM 进行训练，分类.该建议有助于减少 falase-positive 眨眼检测结果，提升整体的精度.
原文：使用opencv和dlib进行人脸姿态估计(python) - 2018.09.18作者：yuanlulu - CSDN在计算机视觉中，物体的姿态是指相对于相机的相对取向和位置。本文主要参考了 Head Pose Estimation using OpenCV and Dlib - 2016.09.26.人脸姿态估计的目的就是获取人脸相对相机的朝向.人脸姿态估计的思想：旋转三维标准模型一定角度，直到模型上“三维特征点”的“2维投影”，与待测试图像上的特征点（图像上的特征点显然是2维）尽量重合.基于OpenCV和Dlib的头部姿态估计 - AIUAIPython 实现：#!/usr/bin/env python3 import cv2 import numpy as np import dlib import time import math detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat") POINTS_NUM_LANDMARK = 68 # 获取最大的人脸 def _largest_face(dets): if len(dets) == 1: return 0 face_areas = [(det.right()-det.left())*(det.bottom()-det.top()) for det in dets] largest_area = face_areas[0] largest_index = 0 for index in range(1, len(dets)): if face_areas[index] > largest_area : largest_index = index largest_area = face_areas[index] print("largest_face index is {} in {} faces".format(largest_index, len(dets))) return largest_index # 从dlib的检测结果抽取姿态估计需要的点坐标 def get_image_points_from_landmark_shape(landmark_shape): if landmark_shape.num_parts != POINTS_NUM_LANDMARK: print("ERROR:landmark_shape.num_parts-{}".format(landmark_shape.num_parts)) return -1, None # 2D image points. # If you change the image, you need to change vector image_points = np.array([ (landmark_shape.part(30).x, landmark_shape.part(30).y), # Nose tip (landmark_shape.part(8).x, landmark_shape.part(8).y), # Chin (landmark_shape.part(36).x, landmark_shape.part(36).y), # Left eye left corner (landmark_shape.part(45).x, landmark_shape.part(45).y), # Right eye right corne (landmark_shape.part(48).x, landmark_shape.part(48).y), # Left Mouth corner (landmark_shape.part(54).x, landmark_shape.part(54).y) # Right mouth corner ], dtype="double") return 0, image_points # 用dlib检测关键点，返回姿态估计需要的几个点坐标 def get_image_points(img): #gray = cv2.cvtColor( img, cv2.COLOR_BGR2GRAY ) # 图片调整为灰色 dets = detector( img, 0 ) if 0 == len( dets ): print( "ERROR: found no face" ) return -1, None largest_index = _largest_face(dets) face_rectangle = dets[largest_index] landmark_shape = predictor(img, face_rectangle) return get_image_points_from_landmark_shape(landmark_shape) # 获取旋转向量和平移向量 def get_pose_estimation(img_size, image_points ): # 3D model points. model_points = np.array([ (0.0, 0.0, 0.0), # Nose tip (0.0, -330.0, -65.0), # Chin (-225.0, 170.0, -135.0), # Left eye left corner (225.0, 170.0, -135.0), # Right eye right corne (-150.0, -150.0, -125.0), # Left Mouth corner (150.0, -150.0, -125.0) # Right mouth corner ]) # Camera internals focal_length = img_size[1] center = (img_size[1]/2, img_size[0]/2) camera_matrix = np.array( [[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]], dtype = "double" ) print("Camera Matrix :{}".format(camera_matrix)) dist_coeffs = np.zeros((4,1)) # Assuming no lens distortion (success, rotation_vector, translation_vector) = cv2.solvePnP(model_points, image_points, camera_matrix, dist_coeffs, flags=cv2.SOLVEPNP_ITERATIVE ) print("Rotation Vector:\n {}".format(rotation_vector)) print("Translation Vector:\n {}".format(translation_vector)) return success, rotation_vector, translation_vector, camera_matrix, dist_coeffs # 从旋转向量转换为欧拉角 def get_euler_angle(rotation_vector): # calculate rotation angles theta = cv2.norm(rotation_vector, cv2.NORM_L2) # transformed to quaterniond w = math.cos(theta / 2) x = math.sin(theta / 2)*rotation_vector[0][0] / theta y = math.sin(theta / 2)*rotation_vector[1][0] / theta z = math.sin(theta / 2)*rotation_vector[2][0] / theta ysqr = y * y # pitch (x-axis rotation) t0 = 2.0 * (w * x + y * z) t1 = 1.0 - 2.0 * (x * x + ysqr) print('t0:{}, t1:{}'.format(t0, t1)) pitch = math.atan2(t0, t1) # yaw (y-axis rotation) t2 = 2.0 * (w * y - z * x) if t2 > 1.0: t2 = 1.0 if t2 < -1.0: t2 = -1.0 yaw = math.asin(t2) # roll (z-axis rotation) t3 = 2.0 * (w * z + x * y) t4 = 1.0 - 2.0 * (ysqr + z * z) roll = math.atan2(t3, t4) print('pitch:{}, yaw:{}, roll:{}'.format(pitch, yaw, roll)) # 单位转换：将弧度转换为度 Y = int((pitch/math.pi)*180) X = int((yaw/math.pi)*180) Z = int((roll/math.pi)*180) return 0, Y, X, Z def get_pose_estimation_in_euler_angle(landmark_shape, im_szie): try: ret, image_points = get_image_points_from_landmark_shape(landmark_shape) if ret != 0: print('get_image_points failed') return -1, None, None, None ret, rotation_vector, translation_vector, camera_matrix, dist_coeffs \ = get_pose_estimation(im_szie, image_points) if ret != True: print('get_pose_estimation failed') return -1, None, None, None ret, pitch, yaw, roll = get_euler_angle(rotation_vector) if ret != 0: print('get_euler_angle failed') return -1, None, None, None euler_angle_str = 'Y:{}, X:{}, Z:{}'.format(pitch, yaw, roll) print(euler_angle_str) return 0, pitch, yaw, roll except Exception as e: print('get_pose_estimation_in_euler_angle exception:{}'.format(e)) return -1, None, None, None if __name__ == '__main__': # rtsp://admin:ts123456@10.20.21.240:554 cap = cv2.VideoCapture(0) while (cap.isOpened()): start_time = time.time() # Read Image ret, im = cap.read() if ret != True: print('read frame failed') continue size = im.shape if size[0] > 700: h = size[0] / 3 w = size[1] / 3 im = cv2.resize( im, (int(w), int(h)), interpolation=cv2.INTER_CUBIC ) size = im.shape ret, image_points = get_image_points(im) if ret != 0: print('get_image_points failed') continue ret, rotation_vector, translation_vector, camera_matrix, dist_coeffs \ = get_pose_estimation(size, image_points) if ret != True: print('get_pose_estimation failed') continue used_time = time.time() - start_time print("used_time:{} sec".format(round(used_time, 3))) ################################ ret, pitch, yaw, roll = get_euler_angle(rotation_vector) euler_angle_str = 'Y:{}, X:{}, Z:{}'.format(pitch, yaw, roll) print(euler_angle_str) ################################ # Project a 3D point (0, 0, 1000.0) onto the image plane. # We use this to draw a line sticking out of the nose (nose_end_point2D, jacobian) = cv2.projectPoints( np.array([(0.0, 0.0, 1000.0)]), rotation_vector, translation_vector, camera_matrix, dist_coeffs) for p in image_points: cv2.circle(im, (int(p[0]), int(p[1])), 3, (0,0,255), -1) p1 = ( int(image_points[0][0]), int(image_points[0][1])) p2 = ( int(nose_end_point2D[0][0][0]), int(nose_end_point2D[0][0][1])) cv2.line(im, p1, p2, (255,0,0), 2) # Display image #cv2.putText( im, str(rotation_vector), (0, 100), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 255), 1 ) cv2.putText( im, euler_angle_str, (0, 120), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 255), 1 ) cv2.imshow("Output", im) cv2.waitKey(1)
原文1 - 利用二维图像进行头部姿态估计 - 2017.10.23原文2 - 人脸姿态估计 - 2018.04.17坐标变换：利用世界坐标系旋转、平移矩阵，将 3D点 从世界坐标系变换到相机坐标系中. 即：通过算法完成世界坐标系(3D)、2D landmark输入image、相机坐标系之间的映射转换和标定.2D人脸特征点与3D世界坐标系的对应点1. 计算旋转矩阵和平移矩阵[基于OpenCV和Dlib的头部姿态估计[译] - AIUAI](https://www.aiuai.cn/aifarm909.html) - 10.2 Python 实现.2. 旋转矩阵转换为欧拉角Yaw:摇头 左正右负Pitch:点头 上负下正Roll:摆头（歪头）左负 右正// C++ static cv::Vec3d RotationMatrix2Euler(const cv::Matx33d& rotation_matrix) { double q0 = sqrt(1 + rotation_matrix(0, 0) + rotation_matrix(1, 1) + rotation_matrix(2, 2)) / 2.0; double q1 = (rotation_matrix(2, 1) - rotation_matrix(1, 2)) / (4.0*q0); double q2 = (rotation_matrix(0, 2) - rotation_matrix(2, 0)) / (4.0*q0); double q3 = (rotation_matrix(1, 0) - rotation_matrix(0, 1)) / (4.0*q0); double t1 = 2.0 * (q0*q2 + q1*q3); double yaw = asin(2.0 * (q0*q2 + q1*q3)); double pitch = atan2(2.0 * (q0*q1 - q2*q3), q0*q0 - q1*q1 - q2*q2 + q3*q3); double roll = atan2(2.0 * (q0*q3 - q1*q2), q0*q0 + q1*q1 - q2*q2 - q3*q3); return cv::Vec3d(pitch, yaw, roll); }% matlab function [euler] = Rot2Euler(R) q0 = sqrt( 1 + R(1,1) + R(2,2) + R(3,3) ) / 2; q1 = (R(3,2) - R(2,3)) / (4*q0) ; q2 = (R(1,3) - R(3,1)) / (4*q0) ; q3 = (R(2,1) - R(1,2)) / (4*q0) ; yaw = asin(2*(q0*q2 + q1*q3)); pitch= atan2(2*(q0*q1-q2*q3), q0*q0-q1*q1-q2*q2+q3*q3); roll = atan2(2*(q0*q3-q1*q2), q0*q0+q1*q1-q2*q2-q3*q3); euler = [pitch, yaw, roll];Python 实现：# Python class PnpHeadPoseEstimator: """ 基于 OpenCV PnP 算法实现的头部姿态估计. It finds Roll, Pitch and Yaw of the head given a figure as input. It uses the PnP algorithm and it requires the dlib library """ def __init__(self, cam_w, cam_h, dlib_shape_predictor_file_path): """ @param cam_w the camera width. If you are using a 640x480 resolution it is 640 @param cam_h the camera height. If you are using a 640x480 resolution it is 480 @dlib_shape_predictor_file_path path to the dlib file for shape prediction (dlib/shape_predictor_68_face_landmarks.dat) """ if(IS_DLIB_INSTALLED == False): raise ValueError('[DEEPGAZE] PnpHeadPoseEstimator: the dlib libray is not installed. \ Please install dlib if you want to use the PnpHeadPoseEstimator class.') if(os.path.isfile(dlib_shape_predictor_file_path)==False): raise ValueError('[DEEPGAZE] PnpHeadPoseEstimator: the files specified do not exist.') # Defining the camera matrix. # To have better result it is necessary to find the focal # lenght of the camera. # fx/fy are the focal lengths (in pixels) # and cx/cy are the optical centres. # These values can be obtained roughly by approximation, # for example in a 640x480 camera: # cx = 640/2 = 320 # cy = 480/2 = 240 # fx = fy = cx/tan(60/2 * pi / 180) = 554.26 c_x = cam_w / 2 c_y = cam_h / 2 f_x = c_x / np.tan(60/2 * np.pi / 180) f_y = f_x # Estimated camera matrix values. self.camera_matrix = np.float32([[f_x, 0.0, c_x], [0.0, f_y, c_y], [0.0, 0.0, 1.0] ]) # These are the camera matrix values estimated on my webcam with # the calibration code (see: src/calibration): # camera_matrix = np.float32([[602.10618226, 0.0, 320.27333589], # [ 0.0, 603.55869786, 229.7537026], # [ 0.0, 0.0, 1.0] ]) # Distortion coefficients self.camera_distortion = np.float32([0.0, 0.0, 0.0, 0.0, 0.0]) # Distortion coefficients estimated by calibration in my webcam # camera_distortion = np.float32([ 0.06232237, -0.41559805, 0.00125389, -0.00402566, 0.04879263]) if(DEBUG==True): print("[DEEPGAZE] PnpHeadPoseEstimator: estimated camera matrix: \n" + str(self.camera_matrix) + "\n") # Declaring the dlib shape predictor object self._shape_predictor = dlib.shape_predictor(dlib_shape_predictor_file_path) def _return_landmarks(self, inputImg, roiX, roiY, roiW, roiH, points_to_return=range(0,68)): """ Return the the roll pitch and yaw angles associated with the input image. @param image It is a colour image. It must be >= 64 pixel. @param radians When True it returns the angle in radians, otherwise in degrees. """ # Creating a dlib rectangle and finding the landmarks dlib_rectangle = dlib.rectangle(left=int(roiX), top=int(roiY), right=int(roiW), bottom=int(roiH)) dlib_landmarks = self._shape_predictor(inputImg, dlib_rectangle) # It selects only the landmarks that # have been indicated in the input parameter "points_to_return". # It can be used in solvePnP() to estimate the 3D pose. landmarks = np.zeros((len(points_to_return),2), dtype=np.float32) counter = 0 for point in points_to_return: landmarks[counter] = [dlib_landmarks.parts()[point].x, dlib_landmarks.parts()[point].y] counter += 1 return landmarks def return_roll_pitch_yaw(self, image, radians=False): """ Return the the roll pitch and yaw angles associated with the input image. @param image It is a colour image. It must be >= 64 pixel. @param radians When True it returns the angle in radians, otherwise in degrees. """ # The dlib shape predictor returns 68 points, # we are interested only in a few of those TRACKED_POINTS = (0, 4, 8, 12, 16, 17, 26, 27, 30, 33, 36, 39, 42, 45, 62) # Antropometric constant values of the human head. # Check the wikipedia EN page and: # "Head-and-Face Anthropometric Survey of U.S. Respirator Users" # # X-Y-Z with X pointing forward and Y on the left and Z up. # The X-Y-Z coordinates used are like the standard # coordinates of ROS (robotic operative system) # OpenCV uses the reference usually used in computer vision: # X points to the right, Y down, Z to the front # # The Male mean interpupillary distance is 64.7 mm # (https://en.wikipedia.org/wiki/Interpupillary_distance) # P3D_RIGHT_SIDE = np.float32([-100.0, -77.5, -5.0]) #0 P3D_GONION_RIGHT = np.float32([-110.0, -77.5, -85.0]) #4 P3D_MENTON = np.float32([0.0, 0.0, -122.7]) #8 P3D_GONION_LEFT = np.float32([-110.0, 77.5, -85.0]) #12 P3D_LEFT_SIDE = np.float32([-100.0, 77.5, -5.0]) #16 P3D_FRONTAL_BREADTH_RIGHT = np.float32([-20.0, -56.1, 10.0]) #17 P3D_FRONTAL_BREADTH_LEFT = np.float32([-20.0, 56.1, 10.0]) #26 P3D_SELLION = np.float32([0.0, 0.0, 0.0]) #27 This is the world origin P3D_NOSE = np.float32([21.1, 0.0, -48.0]) #30 P3D_SUB_NOSE = np.float32([5.0, 0.0, -52.0]) #33 P3D_RIGHT_EYE = np.float32([-20.0, -32.35,-5.0]) #36 P3D_RIGHT_TEAR = np.float32([-10.0, -20.25,-5.0]) #39 P3D_LEFT_TEAR = np.float32([-10.0, 20.25,-5.0]) #42 P3D_LEFT_EYE = np.float32([-20.0, 32.35,-5.0]) #45 #P3D_LIP_RIGHT = np.float32([-20.0, 65.5,-5.0]) #48 #P3D_LIP_LEFT = np.float32([-20.0, 65.5,-5.0]) #54 P3D_STOMION = np.float32([10.0, 0.0, -75.0]) #62 # This matrix contains the 3D points of the # 11 landmarks we want to find. It has been # obtained from antrophometric measurement # of the human head. landmarks_3D = np.float32([P3D_RIGHT_SIDE, P3D_GONION_RIGHT, P3D_MENTON, P3D_GONION_LEFT, P3D_LEFT_SIDE, P3D_FRONTAL_BREADTH_RIGHT, P3D_FRONTAL_BREADTH_LEFT, P3D_SELLION, P3D_NOSE, P3D_SUB_NOSE, P3D_RIGHT_EYE, P3D_RIGHT_TEAR, P3D_LEFT_TEAR, P3D_LEFT_EYE, P3D_STOMION]) # Return the 2D position of our landmarks img_h, img_w, img_d = image.shape landmarks_2D = self._return_landmarks( inputImg=image, roiX=0, roiY=img_w, roiW=img_w, roiH=img_h, points_to_return=TRACKED_POINTS) # Print som red dots on the image # for point in landmarks_2D: # cv2.circle(frame,( point[0], point[1] ), 2, (0,0,255), -1) # Applying the PnP solver to find the 3D pose # of the head from the 2D position of the #landmarks. # retval - bool # rvec - Output rotation vector that, together with tvec, brings # points from the world coordinate system to the camera coordinate system. # tvec - Output translation vector. It is the position of the world origin (SELLION) in camera co-ords retval, rvec, tvec = cv2.solvePnP(landmarks_3D, landmarks_2D, self.camera_matrix, self.camera_distortion) # Get as input the rotational vector # Return a rotational matrix rmat, _ = cv2.Rodrigues(rvec) # euler_angles contain (pitch, yaw, roll) # euler_angles = cv.DecomposeProjectionMatrix( # projMatrix=rmat, # cameraMatrix=camera_matrix, # rotMatrix, # transVect, # rotMatrX=None, # rotMatrY=None, # rotMatrZ=None) head_pose = [rmat[0,0], rmat[0,1], rmat[0,2], tvec[0], rmat[1,0], rmat[1,1], rmat[1,2], tvec[1], rmat[2,0], rmat[2,1], rmat[2,2], tvec[2], 0.0, 0.0, 0.0, 1.0 ] # print(head_pose) #TODO remove this line return self.rotationMatrixToEulerAngles(rmat) # Calculates rotation matrix to euler angles # The result is the same as MATLAB except the order # of the euler angles ( x and z are swapped ). def rotationMatrixToEulerAngles(self, R) : # assert(isRotationMatrix(R)) # To prevent the Gimbal Lock it is possible to use # a threshold of 1e-6 for discrimination sy = math.sqrt(R[0,0] * R[0,0] + R[1,0] * R[1,0]) singular = sy < 1e-6 if not singular : x = math.atan2(R[2,1] , R[2,2]) y = math.atan2(-R[2,0], sy) z = math.atan2(R[1,0], R[0,0]) else : x = math.atan2(-R[1,2], R[1,1]) y = math.atan2(-R[2,0], sy) z = 0 return np.array([x, y, z])上面 python 源码中是采用的 68 个人脸特征点.
原文：Head Pose Estimation using OpenCV and Dlib - 2016.09.26主要介绍如何采用 OpenCV 和 Dlib 库，实现图片中人头部姿态估计.在很多场景中，需要知道人头部相对于相机的倾斜程度. 例如，在 VR 领域，可以采用头部姿态来渲染场景的视角. 在辅助驾驶系统中，对准司机脸部的相机，可以根据头部姿态估计，判断司机是否注意力集中. 此外，还可以采用给予手势的头部姿态来控制 hands-free 应用(或游戏). 例如，将头部从左到有扭转，可以表示 NO. 但，如果是在印度南部，这个信号表示 YES.本文作者认为，头部姿态估计是很有应用价值的.1. 什么是姿态估计？在 CV 领域中，物体的姿态是其相对于相机的方向(orientation)和位置(position). 通过移动物体相对于相机的位置，或者移动相机相对于物体的位置，即可改变物体的姿态.这里所描述的姿态估计问题通常是 Perspective-n-Point (PNP) 问题，其旨在，对于标定相机(calibrated camera)，找出物体的姿态；其中，已知物体的 n 个 3D 点，以及其在图片中所对应的 2D 投影.对于一个标定好的摄像机，根据目标在 3D 世界坐标系中 n 个点的坐标和对应投影到 2D 图像坐标系中的点集之间的变换关系矩阵进行求解.2. 相机运动(camera motion) 的数学描述3D 刚体仅有两种相对于相机的运动：[1] - 平移将相机从其当前 3D 位置 $(X, Y, Z)$ 移动到新的 3D 位置 $(X', Y', Z')$，即为 平移. 易知，其有 3 个自由度(degrees of freedom)，可以分别在 X, Y 或 Z 方向进行评议.平移可以表示为向量：$\mathbf{t} = (X'-X, Y'-Y, Z'-Z)$.[2] - 旋转关于 X, Y 和 Z 轴，旋转相机. 因此，旋转 也有三个自由度. 旋转有很多种表示方式，可以是欧拉角(Euler angles) (roll,pitch,yaw) 或者，3x3 旋转矩阵(rotation matrix)，或者旋转和角度的方向(direction of rotation (i.e. axis ) and angle).因此，3D 物体的姿态估计，即是计算 6 个变量 - 三个平移和三个旋转.3. 姿态估计所需信息为了计算图片中物体的 3D 姿态，需要知道一下信息：[1] - 一些点的 2D 坐标位置图片中一些点的 2D 坐标 (x, y).对于人脸而言，往往选择眼角(corners of the eyes), 鼻尖(tip of the nose), 嘴角(corners of the mouth) 等位置的点. Dlib 库的 人脸关键点检测 可以定位很多人脸关键点.这里采用的人脸关键点有：鼻尖，下巴，左眼的左角，右眼的右角，左嘴角，右嘴角.[2] - 同一批点的 3D 位置同时还需要 2D 特征点对应的 3D 位置. 可能首先会想到，需要获取图片中人体的 3D 模型，才能得到 3D 位置. 而，实际上，是不需要的. 通用 3D 模型即可.如何获取头部的 3D 模型呢？ 实际上不需要 完整的 3D 模型，只需要任意坐标系中一些点的 3D 位置. 这里采用如下的 3D 点：(1) - 鼻尖：( 0.0, 0.0, 0.0)(2) - 下巴：( 0.0, -330.0, -65.0)(3) - 左眼左角：(-225.0f, 170.0f, -135.0)(4) - 右眼右角：( 225.0, 170.0, -135.0)(5) - 左嘴角：(-150.0, -150.0, -125.0)(6) - 右嘴角：(150.0, -150.0, -125.0)要注意的是，这些点是在任意坐标系统的. 即世界坐标系(World Coordinates).[3] - 相机的内在参数(intrinsic parameters)正如上面所述，这里需要首先进行相机标定的(calibrated). 即，需要知道相机的焦距(focal length)，图像的光学中心(optical center)和径向畸变参数(radial distortion parameters).由于不是采用精度 3D 模型来逼近问题，可以采用图片中心(image center)来逼近光学中心(optical center)，采用图像的宽度(像素值)来逼近焦距(focal length)，假设径向畸变(radial disortion) 不存在的话.进行相机参数估计时首先需要对相机进行标定，精确的相机标定需要使用张正友的棋盘格标定，这里还是进行近似. 相机的内参数矩阵需要设定相机的焦距、图像的中心位置并且假设不存在径向畸变. 这里设置相机焦距为图像的宽度(以像素为单位)，图像中心位置为(image.width/2,image.height/2).From: https://blog.csdn.net/cdknight_happy/article/details/799750604. 姿态估计算法原理姿态估计有很多常用算法，最早的算法可以追溯到 1841 年. 这里不涉及到这些算法细节的解释，只是利用其思想.这里需要用到三个坐标系统. 人脸特征的 3D 坐标是以世界坐标系进行表示的，如果旋转和平移是已知的(例如，pose)，那么可以将世界坐标(world coordinates)中的 3D 点转换到相机坐标系(camera coordinates)中的 3D 点. 然后，采用相机的内在参数(焦距、光学中心等)，将相机坐标系中的 3D 点再投影到图像平面(如，图像坐标系).下面将图像成像公式分解，以理解上面的坐标系统的工作原理. 如上图中，$o$ 是相机中心，图中的平面即为图像平面. 目的是，找出 3D 点 $P$ 到图像平面的投影 $p$ 的等式.假设，已知世界坐标系中的 3D 点 $P$ 的位置 $(U, V, W)$. 如果还知道相对于相机坐标系中的世界坐标系的旋转矩阵 $\mathbf{R} $ (3x3 矩阵)和平移向量 $\mathbf{t}$ (3x1 向量)，则可以采用下面的等式计算相机坐标系中点 $P$ 的坐标 $(X, Y, Z)$:其展开形式类似于：如果具备线性代数知识，可以看出上述等式中，如果变量数(如，$(X, Y, Z)$ 和 $(U, V, W)$ )已知的足够多，该等式是线性系统的等式. 其中，$r_{ij}$ 和 $(t_x, t_y, t_z)$ 是未知的，是可以求解未知量.下一章节中会涉及到的，只有 $(X, Y, Z)$ 是已知的，因此该等式不是简单的线性系统问题.5. 直接线性变换虽然已知 3D 模型中的很多点，如$(U, V, W)$，但是不知道 $(X, Y, Z)$. 而仅知道 2D 点的坐标未知，如$(x, y)$. 假设不存在径向畸变(radial disortion)的情况下，图像坐标系中的点 $p$ 的坐标 $(x, y)$ 的计算为：其中，$f_x$ 和 $f_y$ 是在 x 和 y 方向上的焦距长度，$(c_x, c_y)$ 是光学中心. 如果存在径向畸变，则会变得复杂一些. 不过这里先跳过.公式中的 $s$ 是未知的缩放因子. 其在公式中的原因是，在任何图片中深度(depth)都是未知的. 如果将点 $P$ 的 3D 坐标加入到相机的的中心 $o$，则，点 $p$ 是射线 $oP$ 与图像平面的交点. 沿着射线的加入到相机中心和点 $P$ 的所有点，产生相同的图像. 换句话说，采用等式(3)，只能得到 $(X, Y, Z)$，最多还能得到尺度 $s$.这就会使得等式 (2) 变得复杂. 且更类似于：幸运的是，等式(4) 还是可以采用直接线性变换(Direct Linear Transform (DLT)) 方法进行求解的. DLT 方法可以用于求解等式接近与线性系统，但又包含未知尺度因子的等式问题.6. Levenberg-Marquardt OptimizationDLT 方法可能是不够精确的，其主要原因是：[1] - 旋转 $\mathbf{R}$ 包含三个自由度，但 DLT 算法中的矩阵表示要采用 9 个数字. 并没有限制该估计的 3x3 矩阵必须为旋转矩阵.[2] - 更重要的是，DLT 方法并不是对正确的目标函数进行最小化. 理想情况下，想要最小化的是重投影误差(reprojection error). 如等式 (2) 和等式(3) 所示，如果正确的姿态($\mathbf{R}​$ 和 $\mathbf{t} ​$)是已知的，通过将 3D 点投影到 2D 图像，可以预测出 3D 人脸关键点的 2D 位置点.换句话说，如果已知 $\mathbf{R}​$ 和 $\mathbf{t}​$，可以找到每个 3D 点 $P​$ 的投影点 $p​$.另外还知道 2D 人脸特征点(可以采用 Dlib 库或手工选取). 则可以计算投影的 3D 点和 2D 人脸特征点之间的距离. 如果估计的人脸姿态是很完美的，则 3D 点投影到图像平面，会与 2D 人脸特征点高度重合(line up). 如果估计的人脸姿态不够好，则需要计算重投影误差，其计算的是投影的 3D 点和 2D 人脸特征点之间的平方差距离.前面说过，DLT 方法可以用于姿态( $\mathbf{R}$ 和 $\mathbf{t}$)的逼近估计. DLT 方法的一种直接的提升方案是，随机轻微的改变姿态( $\mathbf{R}$ 和 $\mathbf{t}$) ，然后检查重投影误差的衰减情况. 如果是衰减的，则可以接受为新的姿态估计. 然后，继续保持调整 $\mathbf{R}$ 和 $\mathbf{t}$，以寻找更好的姿态估计. 虽然该方案有效，但速度会非常慢. 实践证明， $\mathbf{R}$ 和 $\mathbf{t}$ 迭代改变的有几种主要方式，以使得重投影误差衰减，这种方式被成为 Levenberg-Marquardtoptimization. 详细细节可参考 Wikipedia.7. OpenCV solvePnP 函数OpenCV 中，solvePnP 函数和 solvePnPRansac 函数可以用于估计姿态.solvePnP 函数实现了几种姿态估计算法，可以根据参数 flag 进行选择. 默认 flag 是 SOLVEPNP_ITERATIVE，其是采用 Levenberg-Marquardt 优化的 DTL 方案；SOLVEPNP_P3P 仅采用 3 个点用于计算姿态，只有在采用 solvePnPRansac 函数时才被使用.OpenCV 3中，包含了新的算法，SOLVEPNP_DLS 和 SOLVEPNP_UPNP. 其中，有意思的是，SOLVEPNP_UPNP 还尝试估计相机的内在参数.// C++: bool solvePnP(InputArray objectPoints, InputArray imagePoints, InputArray cameraMatrix, InputArray distCoeffs, OutputArray rvec, OutputArray tvec, bool useExtrinsicGuess=false, int flags=SOLVEPNP_ITERATIVE )# Python: cv2.solvePnP( objectPoints, imagePoints, cameraMatrix, distCoeffs[, rvec[, tvec[, useExtrinsicGuess[, flags] ] ] ] ) → retval, rvec, tvec参数说明：objectPoints - 世界坐标系空间的物体点数组. 一般是 N 个 3D 点的向量(推荐). 也可以是 Nx3 (或 3xN) 的单通道矩阵，或者 Nx1(或 1xN) 的 3 通道矩阵.imagePoints - 对应的图像点数组. 一般可以是 N 个 2D 点的向量. 也可以是 Nx2 (或 2xN) 的单通道矩阵，或者 Nx1(或 1xN) 的 2 通道矩阵.cameraMatrix - 输入的相机矩阵:$$ \begin{bmatrix} f_x & 0 & c_x \\ 0 & f_x & c_y \\ 0 & 0 & 1 \end{bmatrix} $$其中，$f_x$ 和 $f_y$ 可以采用图像的宽度进行逼近，$c_x$ 和 $c_y$ 可以是图像中心的坐标.distCoeffs - 包含 4， 5， 8 或12 个元素的畸变相关系数的输入向量 $(k_1, k_2, p_1, p_2[, k_3[, k_4, k_5, k_6], [s_1, s_2, s_3, s_4]])$ . 如果该向量为 NULL 或空，则假设是 0 畸变的. 通常该值为 NULL. rvec - 输出的旋转向量tvec - 输出的平移向量useExtrinsicGuess - SOLVEPNP_ITERATIVE 涉及的参数. 如果该值为 True，则函数会采用提供的 rvec 和 tvec 值分别作为旋转向量和平移向量的初始化逼近，再进行进一步的优化.flags - PnP 问题求解方法：SOLVEPNP_ITERATIVE Iterative method is based on Levenberg-Marquardt optimization. In this case, the function finds such a pose that minimizes reprojection error, that is the sum of squared distances between the observed projections imagePoints and the projected (using projectPoints() ) objectPoints .SOLVEPNP_P3P Method is based on the paper of X.S. Gao, X.-R. Hou, J. Tang, H.-F. Chang “Complete Solution Classification for the Perspective-Three-Point Problem”. In this case, the function requires exactly four object and image points.SOLVEPNP_EPNP Method has been introduced by F.Moreno-Noguer, V.Lepetit and P.Fua in the paper “EPnP: Efficient Perspective-n-Point Camera Pose Estimation”.The flags below are only available for OpenCV 3SOLVEPNP_DLS Method is based on the paper of Joel A. Hesch and Stergios I. Roumeliotis. “A Direct Least-Squares (DLS) Method for PnP”.SOLVEPNP_UPNP Method is based on the paper of A.Penate-Sanchez, J.Andrade-Cetto, F.Moreno-Noguer. “Exhaustive Linearization for Robust Camera Pose and Focal Length Estimation”. In this case the function also estimates the parameters f_x and f_y assuming that both have the same value. Then the cameraMatrix is updated with the estimated focal length.8. OpenCV solvePnPRansac 函数solvePnPRansac 函数与 solvePnP 函数非常相似，除了其采用了 Random Sample Consensus ( RANSAC ) 以进行更鲁棒的姿态估计.// C++: void solvePnPRansac(InputArray objectPoints, InputArray imagePoints, InputArray cameraMatrix, InputArray distCoeffs, OutputArray rvec, OutputArray tvec, bool useExtrinsicGuess=false, int iterationsCount=100, float reprojectionError=8.0, int minInliersCount=100, OutputArray inliers=noArray(), int flags=ITERATIVE)# Python: cv2.solvePnPRansac( objectPoints, imagePoints, cameraMatrix, distCoeffs[, rvec[, tvec[, useExtrinsicGuess[, iterationsCount[, reprojectionError[, minInliersCount[, inliers[, flags] ] ] ] ] ] ] ] ) → rvec, tvec, inliers新增参数：iterationsCount – The number of times the minimum number of points are picked and the parameters estimated.reprojectionError – As mentioned earlier in RANSAC the points for which the predictions are close enough are called “inliers”. This parameter value is the maximum allowed distance between the observed and computed point projections to consider it an inlier.minInliersCount – Number of inliers. If the algorithm at some stage finds more inliers than minInliersCount , it finishes.inliers – Output vector that contains indices of inliers in objectPoints and imagePoints .9. OpenCV POSITOpenCV 早期采用了 POSIT 的姿态估计算法，其是 C 代码的 API (cvPosit)，并不是 C++ API 的一部分.10. OpenCV 姿态估计的实现源码人脸特征点的位置是硬编码的，如果需要使用自定义的图片，需要修改 image_points 向量.headPose.jpg10.1. C++ 实现#include <opencv2/opencv.hpp> using namespace std; using namespace cv; int main(int argc, char **argv) { // Read input image cv::Mat im = cv::imread("headPose.jpg"); // 2D image points. If you change the image, you need to change vector std::vector<cv::Point2d> image_points; image_points.push_back( cv::Point2d(359, 391) ); // Nose tip image_points.push_back( cv::Point2d(399, 561) ); // Chin image_points.push_back( cv::Point2d(337, 297) ); // Left eye left corner image_points.push_back( cv::Point2d(513, 301) ); // Right eye right corner image_points.push_back( cv::Point2d(345, 465) ); // Left Mouth corner image_points.push_back( cv::Point2d(453, 469) ); // Right mouth corner // 3D model points. std::vector<cv::Point3d> model_points; model_points.push_back(cv::Point3d(0.0f, 0.0f, 0.0f)); // Nose tip model_points.push_back(cv::Point3d(0.0f, -330.0f, -65.0f)); // Chin model_points.push_back(cv::Point3d(-225.0f, 170.0f, -135.0f)); // Left eye left corner model_points.push_back(cv::Point3d(225.0f, 170.0f, -135.0f)); // Right eye right corner model_points.push_back(cv::Point3d(-150.0f, -150.0f, -125.0f)); // Left Mouth corner model_points.push_back(cv::Point3d(150.0f, -150.0f, -125.0f)); // Right mouth corner // Camera internals double focal_length = im.cols; // Approximate focal length. Point2d center = cv::Point2d(im.cols/2,im.rows/2); cv::Mat camera_matrix = (cv::Mat_<double>(3,3) << focal_length, 0, center.x, 0 , focal_length, center.y, 0, 0, 1); cv::Mat dist_coeffs = cv::Mat::zeros(4,1,cv::DataType<double>::type); // Assuming no lens distortion cout << "Camera Matrix " << endl << camera_matrix << endl ; // Output rotation and translation cv::Mat rotation_vector; // Rotation in axis-angle form cv::Mat translation_vector; // Solve for pose cv::solvePnP(model_points, image_points, camera_matrix, dist_coeffs, rotation_vector, translation_vector); // Project a 3D point (0, 0, 1000.0) onto the image plane. // We use this to draw a line sticking out of the nose vector<Point3d> nose_end_point3D; vector<Point2d> nose_end_point2D; nose_end_point3D.push_back(Point3d(0,0,1000.0)); projectPoints(nose_end_point3D, rotation_vector, translation_vector, camera_matrix, dist_coeffs, nose_end_point2D); for(int i=0; i < image_points.size(); i++) { circle(im, image_points[i], 3, Scalar(0,0,255), -1); } cv::line(im,image_points[0], nose_end_point2D[0], cv::Scalar(255,0,0), 2); cout << "Rotation Vector " << endl << rotation_vector << endl; cout << "Translation Vector" << endl << translation_vector << endl; cout << nose_end_point2D << endl; // Display image. cv::imshow("Output", im); cv::waitKey(0); }10.2. Python 实现#!/usr/bin/env python import cv2 import numpy as np import matplotlib.pyplot as plt # Read Image im = cv2.imread("headPose.jpg") size = im.shape # 2D image points. If you change the image, you need to change vector image_points = np.array([ (359, 391), # Nose tip (399, 561), # Chin (337, 297), # Left eye left corner (513, 301), # Right eye right corne (345, 465), # Left Mouth corner (453, 469) # Right mouth corner ], dtype="double") # 3D model points. model_points = np.array([ (0.0, 0.0, 0.0), # Nose tip (0.0, -330.0, -65.0), # Chin (-225.0, 170.0, -135.0), # Left eye left corner (225.0, 170.0, -135.0), # Right eye right corne (-150.0, -150.0, -125.0), # Left Mouth corner (150.0, -150.0, -125.0) # Right mouth corner ]) # Camera internals focal_length = size[1] center = (size[1] / 2, size[0] / 2) camera_matrix = np.array( [[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]], dtype="double") print("Camera Matrix :\n {0}".format(camera_matrix)) dist_coeffs = np.zeros((4, 1)) # Assuming no lens distortion (success, rotation_vector, translation_vector) = \ cv2.solvePnP(model_points, image_points, camera_matrix, dist_coeffs, flags=cv2.cv2.SOLVEPNP_ITERATIVE) # print("Rotation Vector:\n {0}".format(rotation_vector)) print("Translation Vector:\n {0}".format(translation_vector)) # Project a 3D point (0, 0, 1000.0) onto the image plane. # We use this to draw a line sticking out of the nose (nose_end_point2D, jacobian) = cv2.projectPoints( np.array([(0.0, 0.0, 1000.0)]), rotation_vector, translation_vector, camera_matrix, dist_coeffs) for p in image_points: cv2.circle(im, (int(p[0]), int(p[1])), 3, (0, 0, 255), -1) p1 = (int(image_points[0][0]), int(image_points[0][1])) p2 = (int(nose_end_point2D[0][0][0]), int(nose_end_point2D[0][0][1])) cv2.line(im, p1, p2, (255, 0, 0), 2) # Display image plt.imshow(im[:, :,::-1]) plt.axis("off") plt.show()输入如：Camera Matrix : [[1.20e+03 0.00e+00 6.00e+02] [0.00e+00 1.20e+03 3.37e+02] [0.00e+00 0.00e+00 1.00e+00]] Rotation Vector: [[-0.0581844 ] [ 2.20231074] [ 0.01887552]] Translation Vector: [[ 449.53365497] [ -95.72475146] [-2344.07917989]]
<Github - zeusees/FaceBoxes>主页： 智云视图<Github - AIHGF/FaceBoxes>FaceBoxes 完整复现. 论文： FaceBoxes: A CPU Real-time Face Detector with High Accuracy - 2018论文源码：<Github - sfzhang15/FaceBoxes>1. FaceBoxes 测试Caffemodel - FaceBoxes_1024x1024.caffemodel (3.6M)Prototxt - faceboxes_deploy.prototxtCaffe - [caffe-ssd]()# -*- coding: utf-8 -* import numpy as np import matplotlib.pyplot as plt import sys,os import cv2 caffe_root = '/path/to/caffe-ssd/' sys.path.insert(0, caffe_root + 'python') import caffe import time net_file= 'faceboxes_deploy.prototxt' caffe_model='FaceBoxes_1024x1024.caffemodel' caffe.set_mode_cpu() # cpu # caffe.set_mode_gpu() # gpu net = caffe.Net(net_file,caffe_model,caffe.TEST) CLASSES = ('background', 'face') transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape}) transformer.set_transpose('data', (2, 0, 1)) transformer.set_mean('data', np.array([104, 117, 123])) # mean pixel def preprocess(src): img = cv2.resize(src, (1024,1024)) img = img - 127.5 img = img * 0.007843 return img def postprocess(img, out): h = img.shape[0] w = img.shape[1] box = out['detection_out'][0,0,:,3:7] * np.array([w, h, w, h]) cls = out['detection_out'][0,0,:,1] conf = out['detection_out'][0,0,:,2] return (box.astype(np.int32), conf, cls) def detect(imgfile): origimg = cv2.imread(imgfile) transformed_image = transformer.preprocess('data', origimg) net.blobs['data'].data[...] = transformed_image time_start = time.time() out = net.forward() print("[INFO]timecost: ", time.time() - time_start) box, conf, cls = postprocess(origimg, out) for i in range(len(box)): p1 = (box[i][0], box[i][1]) p2 = (box[i][2], box[i][3]) cv2.rectangle(origimg, p1, p2, (0, 255, 0)) p3 = (max(p1[0], 15), max(p1[1], 15)) title = "%s:%.2f" % (CLASSES[int(cls[i])], conf[i]) cv2.putText(origimg, title, p3, cv2.FONT_ITALIC, 0.6, (0, 255, 0), 1) plt.imshow(origimg[:,:,::-1]) plt.axis("off") plt.show() return True if __name__ == '__main__': test_dir = "/path/to/images" for img_name in os.listdir(test_dir): img_file = os.path.join(test_dir, img_name) result = detect(img_file) print("[INFO]Done.")如(会出现漏检)：('[INFO]cpu timecost: ', 0.1766371726989746) ('[INFO]cpu timecost: ', 0.13269805908203125) ('[INFO]gpu timecost: ', 0.024658203125) ('[INFO]gpu timecost: ', 0.01454019546508789)
Dlib 官网 - Dlib C++ LibraryDlib - GithubDlib 是一个十分优秀好用的机器学习库，其源码均由 C++ 实现，并提供了 Python 接口，可广泛适用于很多场景.这里主要记录 Dlib 中关于人脸检测和人脸关键点等技术的 python 应用.pip 安装：sudo apt-get install cmake sudo pip install dlib或 Github 源码安装.1. 人脸检测 Face Detector人脸检测，是检测出图片中包含的正面人脸.1.1. 基于 HOG 特征和线性分类器的人脸检测下面是采用经典的 HOG(Histogram of Oriented Gradients) 特征 结合线性分类器、图像金字塔(image pyramid) 及滑窗检测机制(sliding window detection scheme)实现的人脸检测器.#!/usr/bin/python # -*- coding: utf-8 -*- import sys import dlib detector = dlib.get_frontal_face_detector() def face_detect(imgfile): win = dlib.image_window() print("Processing file: {}".format(imgfile)) img = dlib.load_rgb_image(imgfile) # 人脸检测 dets = detector(img, 1) # len(dets) 即为检测到的人脸个数 print("Number of faces detected: {}".format(len(dets))) # 遍历所有检测到人脸的坐标 # left：人脸左边距离图片左边界的距离 # right：人脸右边距离图片左边界的距离 # top：人脸上边距离图片上边界的距离 # bottom：人脸下边距离图片上边界的距离 for i, d in enumerate(dets): print("Detection {}: Left: {} Top: {} Right: {} Bottom: {}".format( i, d.left(), d.top(), d.right(), d.bottom())) win.clear_overlay() win.set_image(img) win.add_overlay(dets) dlib.hit_enter_to_continue() def face_detect_with_scores(imgfile): win = dlib.image_window() # 如果要进一步得到更全面的检测信息，如每个检测结果的分数(score). # score 值越大，则检测的可信度越高. img = dlib.load_rgb_image(imgfile) # 第三个参数 -1 用于设置调整检测阈值； # 如果该参数值为负数，则返回更多的结果； # 如果该参数值为正数，则返回较少的结果. dets, scores, idx = detector.run(img, 1, -1) for i, d in enumerate(dets): print("Detection {}, score: {}, face_type:{}".format( d, scores[i], idx[i])) win.clear_overlay() win.set_image(img) win.add_overlay(dets) dlib.hit_enter_to_continue() if __name__ == '__main__': imgfile = "test.jpg" face_detect(imgfile) face_detect_with_scores(imgfile)1.2. 基于 CNN 的人脸检测采用预训练的 CNN 模型进行图片中的人脸检测.基于 CNN 模型比基于 HOG 特征模型的人脸检测准确度更高. 但是需要更多的计算资源，即在 GPU 上运行才可有较好的运行速率.预训练模型：http://dlib.net/files/mmod_human_face_detector.dat.bz2#!/usr/bin/python # -*- coding: utf-8 -*- import sys import dlib # 加载预训练人脸检测CNN模型 cnn_face_model = "mmod_human_face_detector.dat" cnn_face_detector = dlib.cnn_face_detection_model_v1(cnn_face_model) def cnn_face_detect(imgfile): win = dlib.image_window() print("Processing file: {}".format(imgfile)) img = dlib.load_rgb_image(imgfile) # CNN 人脸检测器 # 检测器返回 mmod_rectangles object，其包含 mmod_rectangle objects 列表. # mmod_rectangle object 包含两个变量： # [1]. dlib.rectangle object # [2]. confidence score dets = cnn_face_detector(img, 1) # 批量检测图片 # dets = cnn_face_detector([image list], upsample_num, batch_size = 128) # len(dets) 即为检测到的人脸个数 print("Number of faces detected: {}".format(len(dets))) for i, d in enumerate(dets): print("Detection {}: Left: {} Top: {} Right: {} Bottom: {} Confidence: {}".format( i, d.rect.left(), d.rect.top(), d.rect.right(), d.rect.bottom(), d.confidence)) rects = dlib.rectangles() rects.extend([d.rect for d in dets]) win.clear_overlay() win.set_image(img) win.add_overlay(rects) dlib.hit_enter_to_continue() if __name__ == '__main__': imgfile = "test.jpg" cnn_face_detect(imgfile)2. 人脸关键点检测 Face Landmark Detection人脸关键点检测，首先需要检测出图片中的人脸，并估计人脸的关键点姿态(pose). 人脸关键点共有 68 个，分别是人脸各部位的点，如嘴角(corners of the mouth)，眼睛边(corners of the mouth)等.From：https://blog.csdn.net/kgzhang/article/details/753093952.1. HOG 人脸框及CNN人脸关键点检测人脸关键点检测预训练模型：http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2iBUG 300-W 人脸关键点数据集：https://ibug.doc.ic.ac.uk/resources/facial-point-annotations/#!/usr/bin/python # -*- coding: utf-8 -*- import sys import os import dlib # 加载人脸检测器 detector = dlib.get_frontal_face_detector() # 加载人脸关键点检测模型 predictor_path = "shape_predictor_68_face_landmarks.dat" predictor = dlib.shape_predictor(predictor_path) def face_landmark_detect(imgfile): win = dlib.image_window() print("Processing file: {}".format(imgfile)) img = dlib.load_rgb_image(imgfile) win.clear_overlay() win.set_image(img) # 检测每个人脸的边界框 dets = detector(img, 1) # len(dets) 是检测到的人脸数量 print("Number of faces detected: {}".format(len(dets))) for k, d in enumerate(dets): print("Detection {}: Left: {} Top: {} Right: {} Bottom: {}".format( k, d.left(), d.top(), d.right(), d.bottom())) # 检测 box d 内的人脸关键点 shape = predictor(img, d) print("Part 0: {}, Part 1: {} ...".format(shape.part(0), shape.part(1))) # 画出人脸关键点 win.add_overlay(shape) win.add_overlay(dets) dlib.hit_enter_to_continue() if __name__ == '__main__': imgfile = "test.jpg" face_landmark_detect(imgfile)2.2. CNN 人脸框及人脸关键点检测#!/usr/bin/python # -*- coding: utf-8 -*- import sys import os import dlib # 加载预训练人脸检测CNN模型 cnn_face_model = "mmod_human_face_detector.dat" cnn_face_detector = dlib.cnn_face_detection_model_v1(cnn_face_model) # 加载人脸关键点检测模型 predictor_path = "shape_predictor_68_face_landmarks.dat" predictor = dlib.shape_predictor(predictor_path) def cnn_face_landmark_detect(imgfile): win = dlib.image_window() print("Processing file: {}".format(imgfile)) img = dlib.load_rgb_image(imgfile) win.clear_overlay() win.set_image(img) # 检测每个人脸的边界框 dets = cnn_face_detector(img, 1) # len(dets) 是检测到的人脸数量 print("Number of faces detected: {}".format(len(dets))) for i, d in enumerate(dets): print("Detection {}: Left: {} Top: {} Right: {} Bottom: {} Confidence: {}".format( i, d.rect.left(), d.rect.top(), d.rect.right(), d.rect.bottom(), d.confidence)) # 检测 box i 内的人脸关键点 shape = predictor(img, d.rect) print("Part 0: {}, Part 1: {} ...".format(shape.part(0), shape.part(1))) # 画出人脸关键点 win.add_overlay(shape) rects = dlib.rectangles() rects.extend([d.rect for d in dets]) win.add_overlay(rects) dlib.hit_enter_to_continue() if __name__ == '__main__': imgfile = "test.jpg" cnn_face_landmark_detect(imgfile)2.3. 人脸关键点检测 CNN 模型训练这里主要是介绍对于论文 One Millisecond Face Alignment with an Ensemble of Regression Trees(CVPR 2014) 采用 dlib 的实现.基于小规模数据集的人脸关键点模型训练. 假设数据集路径为：examples/faces_folderhttp://dlib.net/train_shape_predictor.py.html#!/usr/bin/python import os import sys import glob import dlib def train_face_landmark_model(faces_folder): options = dlib.shape_predictor_training_options() options.oversampling_amount = 300 options.nu = 0.05 options.tree_depth = 2 options.be_verbose = True # dlib.train_shape_predictor() 函数进行模型训练， # 同时保存最终的 predictor - predictor.dat. # 训练输入是 XML 文件，其包含了训练数据集的图片列表和人脸的位置. training_xml_path = os.path.join(faces_folder, "training_with_face_landmarks.xml") dlib.train_shape_predictor(training_xml_path, "predictor.dat", options) # dlib.test_shape_predictor() 函数测试模型标签， # 其度量 shape_predictor 输出人脸关键点值和 groundtruth 数据间的平均距离. print("\nTraining accuracy: {}".format( dlib.test_shape_predictor(training_xml_path, "predictor.dat"))) # 测试数据集仅用于评估训练模型的表现，而不用于模型训练. # 虽然是在小规模数据集上进行的训练，但结果仍然相当好. # 如果在更大规模的人脸关键点数据集上进行训练，会得到更好的结果. testing_xml_path = os.path.join(faces_folder, "testing_with_face_landmarks.xml") print("Testing accuracy: {}".format( dlib.test_shape_predictor(testing_xml_path, "predictor.dat"))) # 模型应用 # 首先从磁盘加载训练的模型； # 还需要采用人脸检测器检测人脸位置. predictor = dlib.shape_predictor("predictor.dat") detector = dlib.get_frontal_face_detector() # 运行人脸检测，及人脸关键点检测，并显示结果. print("Showing detections and predictions on the images in the faces folder...") win = dlib.image_window() for f in glob.glob(os.path.join(faces_folder, "*.jpg")): print("Processing file: {}".format(f)) img = dlib.load_rgb_image(f) win.clear_overlay() win.set_image(img) # 检测每个人脸的边界框 dets = detector(img, 1) print("Number of faces detected: {}".format(len(dets))) for k, d in enumerate(dets): print("Detection {}: Left: {} Top: {} Right: {} Bottom: {}".format( k, d.left(), d.top(), d.right(), d.bottom())) # 得到第 d 个框的人脸关键点 shape = predictor(img, d) print("Part 0: {}, Part 1: {} ...".format(shape.part(0), shape.part(1))) # 画出人脸关键点 win.add_overlay(shape) win.add_overlay(dets) dlib.hit_enter_to_continue()