Github项目 - Human Image Gender Classifier
Github - Human Image Gender Classifier论文：Expressive Body Capture: 3D Hands, Face, and Body from a Single Image - CVPR2019主页：https://smpl-x.is.tue.mpg.de/基于OpenPose姿态估计结果，裁剪人体并识别人体性别.示例，如图：1. 项目环境Python 3.7TensorFlow 1.13.1ConfigerCUDA-10.0, cuDNN-7.5, Ubuntu 18.04requirements.txt:imageio==2.5.0 pillow numpy==1.16.3 scikit-image==0.15.0 scikit-learn==0.20.3 scipy==1.2.1 tensorflow==1.15.2安装：pip install git+https://github.com/nghorbani/homogenus模型下载，假设权重文件存放路径homogenus/trained_models/tf：https://smpl-x.is.tue.mpg.de/downloadsHomogenus: Human Image Gender Classifier2. 项目源码项目结构：├── homogenus │ ├── tf │ │ ├── homogenus_infer.py │ │ └── __init__.py │ ├── tools │ │ ├── body_cropper.py │ │ ├── image_tools.py │ │ ├── __init__.py │ │ └── omni_tools.py │ └── trained_models │ └── tf │ ├── checkpoint │ ├── TR02_E02_It_002010.ckpt.data-00000-of-00001 │ ├── TR02_E02_It_002010.ckpt.index │ ├── TR02_E02_It_002010.ckpt.meta │ └── version.txt ├── requirements.txt └── setup.py2.1. 模型预测 homogenus_infer.py#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import tensorflow as tf import numpy as np import os, sys import json import cv2 import glob from homogenus.tools.image_tools import put_text_in_image, fontColors, read_prep_image, save_images from homogenus.tools.body_cropper import cropout_openpose, should_accept_pose from homogenus.tools.omni_tools import makepath class Homogenus_infer(object): def __init__(self, trained_model_dir, sess=None): ''' :param trained_model_dir: the directory where you have put the homogenus TF trained models :param sess: ''' best_model_fname = sorted(glob.glob(os.path.join(trained_model_dir , '*.ckpt.index')), key=os.path.getmtime) if len(best_model_fname): self.best_model_fname = best_model_fname[-1].replace('.index', '') else: raise ValueError('Couldnt find TF trained model in the provided directory --trained_model_dir=%s. Make sure you have downloaded them there.' % trained_model_dir) if sess is None: self.sess = tf.Session() else: self.sess = sess # Load graph. self.saver = tf.train.import_meta_graph(self.best_model_fname+'.meta') self.graph = tf.get_default_graph() self.prepare() def prepare(self): print('Restoring checkpoint %s..' % self.best_model_fname) self.saver.restore(self.sess, self.best_model_fname) def predict_genders(self, images_indir, openpose_indir, images_outdir=None, openpose_outdir=None): ''' Given a directory with images and another directory with corresponding openpose genereated jsons will augment openpose jsons with gender labels. :param images_indir: Input directory of images with common extensions :param openpose_indir: Input directory of openpose jsons :param images_outdir: If given will overlay the detected gender on detected humans that pass the criteria :param openpose_outdir: If given will dump the gendered openpose files in this directory. if not will augment the origianls :return: ''' sys.stdout.write('\nRunning homogenus on --images_indir=%s --openpose_indir=%s\n'%(images_indir, openpose_indir)) im_fnames = [] for img_ext in ['png', 'jpg', 'jpeg', 'bmp']: im_fnames.extend(glob.glob(os.path.join(images_indir, '*.%s'%img_ext))) if len(im_fnames): sys.stdout.write('Found %d images\n' % len(im_fnames)) else: raise ValueError('No images could be found in %s'%images_indir) accept_threshold = 0.9 crop_margin = 0.08 if images_outdir is not None: makepath(images_outdir) if openpose_outdir is None: openpose_outdir = openpose_indir else: makepath(openpose_outdir) Iph = self.graph.get_tensor_by_name(u'input_images:0') probs_op = self.graph.get_tensor_by_name(u'probs_op:0') for im_fname in im_fnames: im_basename = os.path.basename(im_fname) img_ext = im_basename.split('.')[-1] openpose_in_fname = os.path.join(openpose_indir, im_basename.replace('.%s'%img_ext, '_keypoints.json')) with open(openpose_in_fname, 'r') as f: pose_data = json.load(f) im_orig = cv2.imread(im_fname, 3)[:,:,::-1].copy() for opnpose_pIdx in range(len(pose_data['people'])): pose_data['people'][opnpose_pIdx]['gender_pd'] = 'neutral' pose = np.asarray(pose_data['people'][opnpose_pIdx]['pose_keypoints_2d']).reshape(-1, 3) if not should_accept_pose(pose, human_prob_thr=0.5): continue crop_info = cropout_openpose(im_fname, pose, want_image=True, crop_margin=crop_margin) cropped_image = crop_info['cropped_image'] if cropped_image.shape[0] < 200 or cropped_image.shape[1] < 200: continue img = read_prep_image(cropped_image)[np.newaxis] probs_ob = self.sess.run(probs_op, feed_dict={Iph: img})[0] gender_id = np.argmax(probs_ob, axis=0) gender_prob = probs_ob[gender_id] gender_pd = 'male' if gender_id == 0 else 'female' if gender_prob>accept_threshold: color = 'green' text = 'pred:%s[%.3f]' % (gender_pd, gender_prob) else: text = 'thr:%s_pred:%s[%.3f]' % ('neutral', gender_pd, gender_prob) gender_pd = 'neutral' color = 'grey' x1 = crop_info['crop_boundary']['offset_width'] y1 = crop_info['crop_boundary']['offset_height'] x2 = crop_info['crop_boundary']['target_width'] + x1 y2 = crop_info['crop_boundary']['target_height'] + y1 im_orig = cv2.rectangle(im_orig, (x1, y1), (x2, y2), fontColors[color], 2) im_orig = put_text_in_image(im_orig, [text], color, (x1, y1))[0] pose_data['people'][opnpose_pIdx]['gender_pd'] = gender_pd sys.stdout.write('%s -- peron_id %d --> %s\n'%(im_fname, opnpose_pIdx, gender_pd)) if images_outdir != None: save_images(im_orig, images_outdir, [os.path.basename(im_fname)]) openpose_out_fname = os.path.join(openpose_outdir, im_basename.replace('.%s'%img_ext, '_keypoints.json')) with open(openpose_out_fname, 'w') as f: json.dump(pose_data, f) if images_outdir is not None: sys.stdout.write('Dumped overlayed images at %s'%images_outdir) sys.stdout.write('Dumped gendered openpose keypoints at %s'%openpose_outdir) if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() parser.add_argument("-tm", "--trained_model_dir", default="./homogenus/trained_models/tf/", help="The path to the directory holding homogenus trained models in TF.") parser.add_argument("-ii", "--images_indir", required= True, help="Directory of the input images.") parser.add_argument("-oi", "--openpose_indir", required=True, help="Directory of openpose keypoints, e.g. json files.") parser.add_argument("-io", "--images_outdir", default=None, help="Directory to put predicted gender overlays. If not given, wont produce any overlays.") parser.add_argument("-oo", "--openpose_outdir", default=None, help="Directory to put the openpose gendered keypoints. If not given, it will augment the original openpose json files.") ps = parser.parse_args() hg = Homogenus_infer(trained_model_dir=ps.trained_model_dir) hg.predict_genders(images_indir=ps.images_indir, openpose_indir=ps.openpose_indir, images_outdir=ps.images_outdir, openpose_outdir=ps.openpose_outdir)2.2. body_cropper.py#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- from PIL import Image import numpy as np import os import json from homogenus.tools.image_tools import cropout_openpose def should_accept_pose(pose, human_prob_thr=.5): ''' :param pose: :param human_prob_thr: :return: ''' rleg_ids = [12,13] lleg_ids = [9,10] rarm_ids = [5,6,7] larm_ids = [2,3,4] head_ids = [16,15,14,17,0,1] human_prob = pose[:, 2].mean() rleg = sum(pose[rleg_ids][:, 2] > 0.0) lleg = sum(pose[lleg_ids][:, 2] > 0.0) rarm = sum(pose[rarm_ids][:, 2] > 0.0) larm = sum(pose[larm_ids][:, 2] > 0.0) head = sum(pose[head_ids][:, 2] > 0.0) if rleg<1 and lleg<1: return False if rarm<1 and larm<1: return False if head<2: return False if human_prob < human_prob_thr: return False return True def crop_humans(im_fname, pose_fname, want_image=True, human_prob_thr=0.5): ''' :param im_fname: the input image path :param pose_fname: the corresponding openpose json file :param want_image: if False will only return the crop boundaries, otherwise will also return the cropped image :param human_prob_thr: the probability to threshold the detected humans :return: ''' crop_infos = {} with open(pose_fname) as f: pose_data = json.load(f) if not len(pose_data['people']): return crop_infos for pIdx in range(len(pose_data['people'])): pose = np.asarray(pose_data['people'][pIdx]['pose_keypoints_2d']).reshape(-1, 3) if not should_accept_pose(pose, human_prob_thr=human_prob_thr): continue crop_info = cropout_openpose(im_fname, pose, want_image=want_image) crop_infos['%02d'%pIdx] = { 'crop_info':crop_info['crop_boundary'], #'pose':pose, 'pose_hash':hash(pose.tostring())} if want_image: crop_infos['%02d' % pIdx]['cropped_image'] = crop_info['cropped_image'].astype(np.uint8) return crop_infos def crop_dataset(base_dir, dataset_name, human_prob_thr=.5): import glob import random results_dir = os.path.join(base_dir, dataset_name, 'cropped_body_tight') images_dir = os.path.join(base_dir, dataset_name, 'images') pose_jsonpath = os.path.join(base_dir, dataset_name, 'openpose_json') if not os.path.exists(results_dir): os.makedirs(results_dir) crop_infos_jsonpath = os.path.join(results_dir, 'crop_infos.json') fnames = glob.glob(os.path.join(images_dir, '*.jpg')) random.shuffle(fnames) crop_infos = {} for fname in fnames: pose_fname = os.path.join(pose_jsonpath, os.path.basename(fname).replace('.jpg', '_keypoints.json')) if not os.path.exists(pose_fname): continue cur_crop_info = crop_humans(fname, pose_fname, human_prob_thr=human_prob_thr, want_image=True) for pname in cur_crop_info.keys(): crop_id = '%s_%s' % (os.path.basename(fname).split('.')[0], pname) crop_outpath = os.path.join(results_dir, '%s.jpg' % crop_id) cropped_image = cur_crop_info[pname]['cropped_image'] if cropped_image.shape[0] < 200 or cropped_image.shape[1] < 200: continue cur_crop_info.pop('cropped_image') # print(crop_outpath) result = Image.fromarray(cropped_image[:,:,::-1]) result.save(crop_outpath) crop_infos[crop_id] = cur_crop_info with open(crop_infos_jsonpath, 'w') as f: json.dump(crop_infos, f)2.3. image_tools.py#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import cv2 import numpy as np import os fontColors = {'red': (255, 0, 0), 'green': (0, 255, 0), 'yellow': (255, 255, 0), 'blue': (0, 255, 255), 'orange': (255, 165, 0), 'black': (0, 0, 0), 'grey': (169, 169, 169), 'white': (255, 255, 255), } def crop_to_bounding_box(image, offset_height, offset_width, target_height, target_width): cropped = image[offset_height:offset_height + target_height, offset_width:offset_width + target_width, :] return cropped def pad_to_bounding_box(image, offset_height, offset_width, target_height, target_width): height, width, depth = image.shape after_padding_width = target_width - offset_width - width after_padding_height = target_height - offset_height - height # Do not pad on the depth dimensions. paddings = ((offset_height, after_padding_height), (offset_width, after_padding_width), (0, 0)) padded = np.pad(image, paddings, 'constant') return padded def resize_image_with_crop_or_pad(image, target_height, target_width): # crop to ratio, center height, width, c = image.shape width_diff = target_width - width offset_crop_width = max(-width_diff // 2, 0) offset_pad_width = max(width_diff // 2, 0) height_diff = target_height - height offset_crop_height = max(-height_diff // 2, 0) offset_pad_height = max(height_diff // 2, 0) # Maybe crop if needed. # print('image shape', image.shape) cropped = crop_to_bounding_box(image, offset_crop_height, offset_crop_width, min(target_height, height), min(target_width, width)) # print('after cropp', cropped.shape) # Maybe pad if needed. resized = pad_to_bounding_box(cropped, offset_pad_height, offset_pad_width, target_height, target_width) # print('after pad', resized.shape) return resized[:target_height, :target_width, :] def cropout_openpose(im_fname, pose, want_image=True, crop_margin=0.08): im_orig = cv2.imread(im_fname, 3) im_height, im_width = im_orig.shape[0], im_orig.shape[1] pose = pose[pose[:, 2] > 0.0] x_min, x_max = pose[:, 0].min(), pose[:, 0].max() y_min, y_max = pose[:, 1].min(), pose[:, 1].max() margin_h = crop_margin * im_height margin_w = crop_margin * im_width offset_height = int(max((y_min - margin_h), 0)) target_height = int(min((y_max + margin_h), im_height)) - offset_height offset_width = int(max((x_min - margin_w), 0)) target_width = int(min((x_max + margin_w), im_width)) - offset_width crop_info = {'crop_boundary': {'offset_height':offset_height, 'target_height':target_height, 'offset_width':offset_width, 'target_width':target_width}} if want_image: crop_info['cropped_image'] = crop_to_bounding_box(im_orig, offset_height, offset_width, target_height, target_width) return crop_info def put_text_in_image(images, text, color ='white', position=None): ''' :param images: 4D array of images :param text: list of text to be printed in each image :param color: the color or colors of each text :return: ''' import cv2 if not isinstance(text, list): text = [text] if not isinstance(color, list): color = [color for _ in range(images.shape[0])] if images.ndim == 3: images = images.reshape(1,images.shape[0],images.shape[1],3) images_out = [] for imIdx in range(images.shape[0]): img = images[imIdx].astype(np.uint8) font = cv2.FONT_HERSHEY_SIMPLEX if position is None:position = (10, img.shape[1]) fontScale = 1. lineType = 2 fontColor = fontColors[color[imIdx]] cv2.putText(img, text[imIdx], position, font, fontScale, fontColor, lineType) images_out.append(img) return np.array(images_out) def read_prep_image(im_fname, avoid_distortion=True): ''' if min(height, width) is larger than 224 subsample to 224. this will also affect the larger dimension. in the end crop and pad the whole image to get to 224x224 :param im_fname: :return: ''' if isinstance(im_fname, np.ndarray): image_data = im_fname else: image_data = cv2.imread(im_fname, 3) # height, width = image_reader.read_image_dims(sess, image_data) # image_data = image_reader.decode_jpeg(sess, image_data) # print(image_data.min(), image_data.max(), image_data.shape) # import matplotlib.pyplot as plt # plt.imshow(image_data[:,:,::-1].astype(np.uint8)) # plt.show() # height, width = image_data.shape[0], image_data.shape[1] # if min(height, width) > 224: # print(image_data.shape) # rt = 224. / min(height, width) # image_data = cv2.resize(image_data, (int(rt * width), int(rt * height)), interpolation=cv2.INTER_AREA) # print('>>resized to>>',image_data.shape) height, width = image_data.shape[0], image_data.shape[1] if avoid_distortion: if max(height, width) > 224: # print(image_data.shape) rt = 224. / max(height, width) image_data = cv2.resize(image_data, (int(rt * width), int(rt * height)), interpolation=cv2.INTER_AREA) # print('>>resized to>>',image_data.shape) else: from skimage.transform import resize image_data = resize(image_data, (224, 224), mode='constant', anti_aliasing=False, preserve_range=True) # print(image_data.min(), image_data.max(), image_data.shape) # import matplotlib.pyplot as plt # plt.imshow(image_data[:,:,::-1].astype(np.uint8)) # plt.show() image_data = resize_image_with_crop_or_pad(image_data, 224, 224) # print(image_data.min(), image_data.max(), image_data.shape) # import matplotlib.pyplot as plt # plt.imshow(image_data[:, :, ::-1].astype(np.uint8)) # plt.show() #return image_data.astype(np.float32) return image_data.astype(np.uint8) def save_images(images, out_dir, im_names = None): from homogenus.tools.omni_tools import id_generator if images.ndim == 3: images = images.reshape(1,images.shape[0],images.shape[1],3) from PIL import Image if im_names is None: im_names = ['%s.jpg'%id_generator(4) for i in range(images.shape[0])] for imIdx in range(images.shape[0]): result = Image.fromarray(images[imIdx].astype(np.uint8)) result.save(os.path.join(out_dir, im_names[imIdx])) return True2.4. omni_tools.py#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import numpy as np copy2cpu = lambda tensor: tensor.detach().cpu().numpy() colors = { 'pink': [.7, .7, .9], 'purple': [.9, .7, .7], 'cyan': [.7, .75, .5], 'red': [1.0,0.0,0.0], 'green': [.0, 1., .0], 'yellow': [1., 1., 0], 'brown': [.5, .7, .7], 'blue': [.0, .0, 1.], 'offwhite': [.8, .9, .9], 'white': [1., 1., 1.], 'orange': [.5, .65, .9], 'grey': [.7, .7, .7], 'black': np.zeros(3), 'white': np.ones(3), 'yellowg': [0.83,1,0], } def id_generator(size=13): import string import random chars = string.ascii_uppercase + string.digits return ''.join(random.choice(chars) for _ in range(size)) def log2file(logpath=None, auto_newline = True): import sys if logpath is not None: makepath(logpath, isfile=True) fhandle = open(logpath,'a+') else: fhandle = None def _(text): if auto_newline: if not text.endswith('\n'): text = text + '\n' sys.stderr.write(text) if fhandle is not None: fhandle.write(text) fhandle.flush() return lambda text: _(text) def makepath(desired_path, isfile = False): ''' if the path does not exist make it :param desired_path: can be path to a file or a folder name :return: ''' import os if isfile: if not os.path.exists(os.path.dirname(desired_path)): os.makedirs(os.path.dirname(desired_path)) else: if not os.path.exists(desired_path): os.makedirs(desired_path) return desired_path
出处：Paddle文档平台 - ViT（ Vision Transformer）1. ViT模型介绍在计算机视觉领域中，多数算法都是保持CNN整体结构不变，在CNN中增加attention模块或者使用attention模块替换CNN中的某些部分。有研究者提出，没有必要总是依赖于CNN。因此，作者提出ViT[1]算法，仅仅使用Transformer结构也能够在图像分类任务中表现很好。受到NLP领域中Transformer成功应用的启发，ViT算法中尝试将标准的Transformer结构直接应用于图像，并对整个图像分类流程进行最少的修改。具体来讲，ViT算法中，会将整幅图像拆分成小图像块，然后把这些小图像块的线性嵌入序列作为Transformer的输入送入网络，然后使用监督学习的方式进行图像分类的训练。该算法在中等规模（例如ImageNet）以及大规模（例如ImageNet-21K、JFT-300M）数据集上进行了实验验证，发现：Transformer相较于CNN结构，缺少一定的平移不变性和局部感知性，因此在数据量不充分时，很难达到同等的效果。具体表现为使用中等规模的ImageNet训练的Transformer会比ResNet在精度上低几个百分点。当有大量的训练样本时，结果则会发生改变。使用大规模数据集进行预训练后，再使用迁移学习的方式应用到其他数据集上，可以达到或超越当前的SOTA水平。2. ViT模型结构与实现ViT算法的整体结构如 图1 所示。图1 ViT算法结构示意图2.1. 图像分块嵌入考虑到在Transformer结构中，输入是一个二维的矩阵，矩阵的形状可以表示为 $(N,D)$，其中 $N$ 是sequence的长度，而 $D$ 是sequence中每个向量的维度。因此，在ViT算法中，首先需要设法将 $H \times W \times C$ 的三维图像转化为 $(N,D)$ 的二维输入。ViT中的具体实现方式为：将 $H \times W \times C$ 的图像，变为一个 $N \times (P^2 * C)$ 的序列。这个序列可以看作是一系列展平的图像块，也就是将图像切分成小块后，再将其展平。该序列中一共包含了 $N=HW/P^2$ 个图像块，每个图像块的维度则是 $(P^2*C)$。其中 $P$ 是图像块的大小，$C$ 是通道数量。经过如上变换，就可以将 $N$ 视为sequence的长度了。但是，此时每个图像块的维度是 $(P^2*C)$，而我们实际需要的向量维度是 $D$，因此我们还需要对图像块进行 Embedding。这里 Embedding 的方式非常简单，只需要对每个 $(P^2*C)$ 的图像块做一个线性变换，将维度压缩为 $D$ 即可。上述对图像进行分块以及 Embedding 的具体方式如 图2 所示。图2 图像分块嵌入示意图具体代码实现如下所示。本文中将每个大小为 $P$ 的图像块经过大小为 $P$ 的卷积核来代替原文中将大小为 $P$ 的图像块展平后接全连接运算的操作。# 图像分块、Embedding class PatchEmbed(nn.Layer): def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768): super().__init__() # 原始大小为int，转为tuple，即：img_size原始输入224，变换后为[224,224] img_size = to_2tuple(img_size) patch_size = to_2tuple(patch_size) # 图像块的个数 num_patches = (img_size[1] // patch_size[1]) * \ (img_size[0] // patch_size[0]) self.img_size = img_size self.patch_size = patch_size self.num_patches = num_patches # kernel_size=块大小，即每个块输出一个值，类似每个块展平后使用相同的全连接层进行处理 # 输入维度为3，输出维度为块向量长度 # 与原文中：分块、展平、全连接降维保持一致 # 输出为[B, C, H, W] self.proj = nn.Conv2D( in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) def forward(self, x): B, C, H, W = x.shape assert H == self.img_size[0] and W == self.img_size[1], \ "Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})." # [B, C, H, W] -> [B, C, H*W] ->[B, H*W, C] x = self.proj(x).flatten(2).transpose((0, 2, 1)) return x2.2. 多头注意力将图像转化为 $N \times (P^2 * C)$ 的序列后，就可以将其输入到 Transformer 结构中进行特征提取了，如 图3 所示。图3 多头注意力示意图Transformer 结构中最重要的结构就是 Multi-head Attention，即多头注意力结构。具有2个head的 Multi-head Attention 结构如 图4 所示。输入 $a^i$ 经过转移矩阵，并切分生成 $q^{(i,1)}$、$q^{(i,2)}$、$k^{(i,1)}$、$k^{(i,2)}$、$v^{(i,1)}$、$v^{(i,2)}$，然后 $q^{(i,1)}$ 与 $k^{(i,1)}$ 做 attention，得到权重向量 $\alpha$，将 $\alpha$ 与 $v^{(i,1)}$ 进行加权求和，得到最终的 $b^{(i,1)}(i=1,2,…,N)$，同理可以得到 $b^{(i,2)}(i=1,2,…,N)$。接着将它们拼接起来，通过一个线性层进行处理，得到最终的结果。图4 多头注意力其中，使用 $q^{(i,j)}$、$k^{(i,j)}$ 与 $v^{(i,j)}$ 计算 $b^{(i,j)}(i=1,2,…,N)$ 的方法是缩放点积注意力 (Scaled Dot-Product Attention)。 结构如 图5 所示。首先使用每个 $q^{(i,j)}$ 去与 $k^{(i,j)}$ 做 attention，这里说的 attention 就是匹配这两个向量有多接近，具体的方式就是计算向量的加权内积，得到 $\alpha_{(i,j)}$。这里的加权内积计算方式如下所示：$$ \alpha_{(1,i)} = q^1 * k^i / \sqrt{d} $$其中，$d$ 是 $q$ 和 $k$ 的维度，因为 $q*k$ 的数值会随着维度的增大而增大，因此除以 $\sqrt{d}$ 的值也就相当于归一化的效果。接下来，把计算得到的 $\alpha_{(i,j)}$ 取 softmax 操作，再将其与 $v^{(i,j)}$ 相乘。图5 缩放点积注意力具体代码实现如下所示。# Multi-head Attention class Attention(nn.Layer): def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.): super().__init__() self.num_heads = num_heads head_dim = dim // num_heads self.scale = qk_scale or head_dim**-0.5 # 计算 q,k,v 的转移矩阵 self.qkv = nn.Linear(dim, dim * 3, bias_attr=qkv_bias) self.attn_drop = nn.Dropout(attn_drop) # 最终的线性层 self.proj = nn.Linear(dim, dim) self.proj_drop = nn.Dropout(proj_drop) def forward(self, x): N, C = x.shape[1:] # 线性变换 qkv = self.qkv(x).reshape((-1, N, 3, self.num_heads, C // self.num_heads)).transpose((2, 0, 3, 1, 4)) # 分割 query key value q, k, v = qkv[0], qkv[1], qkv[2] # Scaled Dot-Product Attention # Matmul + Scale attn = (q.matmul(k.transpose((0, 1, 3, 2)))) * self.scale # SoftMax attn = nn.functional.softmax(attn, axis=-1) attn = self.attn_drop(attn) # Matmul x = (attn.matmul(v)).transpose((0, 2, 1, 3)).reshape((-1, N, C)) # 线性变换 x = self.proj(x) x = self.proj_drop(x) return x2.3. 多层感知机（MLP）Transformer 结构中还有一个重要的结构就是 MLP，即多层感知机，如 图6 所示。图6 MLP多层感知机的结构多层感知机由输入层、输出层和至少一层的隐藏层构成。网络中各个隐藏层中神经元可接收相邻前序隐藏层中所有神经元传递而来的信息，经过加工处理后将信息输出给相邻后续隐藏层中所有神经元。在多层感知机中，相邻层所包含的神经元之间通常使用“全连接”方式进行连接。多层感知机可以模拟复杂非线性函数功能，所模拟函数的复杂性取决于网络隐藏层数目和各层中神经元数目。多层感知机的结构如 图7 所示。图7 多层感知机具体代码实现如下所示。class Mlp(nn.Layer): def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): super().__init__() out_features = out_features or in_features hidden_features = hidden_features or in_features self.fc1 = nn.Linear(in_features, hidden_features) self.act = act_layer() self.fc2 = nn.Linear(hidden_features, out_features) self.drop = nn.Dropout(drop) def forward(self, x): # 输入层：线性变换 x = self.fc1(x) # 应用激活函数 x = self.act(x) # Dropout x = self.drop(x) # 输出层：线性变换 x = self.fc2(x) # Dropout x = self.drop(x) return x2.4. DropPath除了以上重要模块意外，代码实现过程中还使用了DropPath（Stochastic Depth）来代替传统的Dropout结构，DropPath可以理解为一种特殊的 Dropout。其作用是在训练过程中随机丢弃子图层（randomly drop a subset of layers），而在预测时正常使用完整的 Graph。具体实现如下：def drop_path(x, drop_prob=0., training=False): if drop_prob == 0. or not training: return x keep_prob = paddle.to_tensor(1 - drop_prob) shape = (paddle.shape(x)[0], ) + (1, ) * (x.ndim - 1) random_tensor = keep_prob + paddle.rand(shape, dtype=x.dtype) random_tensor = paddle.floor(random_tensor) output = x.divide(keep_prob) * random_tensor return output class DropPath(nn.Layer): def __init__(self, drop_prob=None): super(DropPath, self).__init__() self.drop_prob = drop_prob def forward(self, x): return drop_path(x, self.drop_prob, self.training)2.5. 基础模块基于上面实现的 Attention、MLP、DropPath模块就可以组合出 Vision Transformer 模型的一个基础模块，如 图8 所示。图8 基础模块示意图基础模块的具体实现如下：class Block(nn.Layer): def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., drop_path=0., act_layer=nn.GELU, norm_layer='nn.LayerNorm', epsilon=1e-5): super().__init__() self.norm1 = eval(norm_layer)(dim, epsilon=epsilon) # Multi-head Self-attention self.attn = Attention( dim, num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop) # DropPath self.drop_path = DropPath(drop_path) if drop_path > 0. else Identity() self.norm2 = eval(norm_layer)(dim, epsilon=epsilon) mlp_hidden_dim = int(dim * mlp_ratio) self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) def forward(self, x): # Multi-head Self-attention， Add， LayerNorm x = x + self.drop_path(self.attn(self.norm1(x))) # Feed Forward， Add， LayerNorm x = x + self.drop_path(self.mlp(self.norm2(x))) return x2.6. 定义ViT网络基础模块构建好后，就可以构建完整的ViT网络了。在构建完整网络结构之前，还需要给大家介绍几个模块：2.6.1. Class Token假设我们将原始图像切分成 $3 \times 3$ 共9个小图像块，最终的输入序列长度却是10，也就是说我们这里人为的增加了一个向量进行输入，我们通常将人为增加的这个向量称为 Class Token。那么这个 Class Token 有什么作用呢？我们可以想象，如果没有这个向量，也就是将 $N=9$ 个向量输入 Transformer 结构中进行编码，我们最终会得到9个编码向量，可对于图像分类任务而言，我们应该选择哪个输出向量进行后续分类呢？因此，ViT算法提出了一个可学习的嵌入向量 Class Token，将它与9个向量一起输入到 Transformer 结构中，输出10个编码向量，然后用这个 Class Token 进行分类预测即可。其实这里也可以理解为：ViT 其实只用到了 Transformer 中的 Encoder，而并没有用到 Decoder，而 Class Token 的作用就是寻找其他9个输入向量对应的类别。2.6.2. Positional Encoding按照 Transformer 结构中的位置编码习惯，这个工作也使用了位置编码。不同的是，ViT 中的位置编码没有采用原版 Transformer 中的 $sincos$ 编码，而是直接设置为可学习的 Positional Encoding。对训练好的 Positional Encoding 进行可视化，如 图9 所示。我们可以看到，位置越接近，往往具有更相似的位置编码。此外，出现了行列结构，同一行/列中的 patch 具有相似的位置编码。图9 Positional Encoding2.6.3. MLP Head得到输出后，ViT中使用了 MLP Head对输出进行分类处理，这里的 MLP Head 由 LayerNorm 和两层全连接层组成，并且采用了 GELU 激活函数。具体代码如下所示。首先构建基础模块部分，包括：参数初始化配置、独立的不进行任何操作的网络层。# 参数初始化配置 trunc_normal_ = nn.initializer.TruncatedNormal(std=.02) zeros_ = nn.initializer.Constant(value=0.) ones_ = nn.initializer.Constant(value=1.) # 将输入 x 由 int 类型转为 tuple 类型 def to_2tuple(x): return tuple([x] * 2) # 定义一个什么操作都不进行的网络层 class Identity(nn.Layer): def __init__(self): super(Identity, self).__init__() def forward(self, input): return input完整代码如下所示。class VisionTransformer(nn.Layer): def __init__(self, img_size=224, patch_size=16, in_chans=3, class_dim=1000, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer='nn.LayerNorm', epsilon=1e-5, **args): super().__init__() self.class_dim = class_dim self.num_features = self.embed_dim = embed_dim # 图片分块和降维，块大小为patch_size，最终块向量维度为768 self.patch_embed = PatchEmbed( img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) # 分块数量 num_patches = self.patch_embed.num_patches # 可学习的位置编码 self.pos_embed = self.create_parameter( shape=(1, num_patches + 1, embed_dim), default_initializer=zeros_) self.add_parameter("pos_embed", self.pos_embed) # 人为追加class token，并使用该向量进行分类预测 self.cls_token = self.create_parameter( shape=(1, 1, embed_dim), default_initializer=zeros_) self.add_parameter("cls_token", self.cls_token) self.pos_drop = nn.Dropout(p=drop_rate) dpr = np.linspace(0, drop_path_rate, depth) # transformer self.blocks = nn.LayerList([ Block( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, epsilon=epsilon) for i in range(depth) ]) self.norm = eval(norm_layer)(embed_dim, epsilon=epsilon) # Classifier head self.head = nn.Linear(embed_dim, class_dim) if class_dim > 0 else Identity() trunc_normal_(self.pos_embed) trunc_normal_(self.cls_token) self.apply(self._init_weights) # 参数初始化 def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight) if isinstance(m, nn.Linear) and m.bias is not None: zeros_(m.bias) elif isinstance(m, nn.LayerNorm): zeros_(m.bias) ones_(m.weight) def forward_features(self, x): B = paddle.shape(x)[0] # 将图片分块，并调整每个块向量的维度 x = self.patch_embed(x) # 将class token与前面的分块进行拼接 cls_tokens = self.cls_token.expand((B, -1, -1)) x = paddle.concat((cls_tokens, x), axis=1) # 将编码向量中加入位置编码 x = x + self.pos_embed x = self.pos_drop(x) # 堆叠 transformer 结构 for blk in self.blocks: x = blk(x) # LayerNorm x = self.norm(x) # 提取分类 tokens 的输出 return x[:, 0] def forward(self, x): # 获取图像特征 x = self.forward_features(x) # 图像分类 x = self.head(x) return x3. 模型指标ViT模型在常用数据集上进行迁移学习，最终指标如 图10 所示。可以看到，在ImageNet上，ViT达到的最高指标为88.55%；在ImageNet ReaL上，ViT达到的最高指标为90.72%；在CIFAR100上，ViT达到的最高指标为94.55%；在VTAB(19 tasks)上，ViT达到的最高指标为88.55%。图10 ViT网络指标4. 模型特点作为CV领域最经典的 Transformer 算法之一，不同于传统的CNN算法，ViT尝试将标准的Transformer结构直接应用于图像，并对整个图像分类流程进行最少的修改。为了满足 Transformer 输入结构的要求，将整幅图像拆分成小图像块，然后把这些小图像块的线性嵌入序列输入到网络。同时，使用了Class Token的方式进行分类预测。5. 参考文献[1] - An Image is Worth 16x16 Words:Transformers for Image Recognition at Scale
基于深度神经网络的噪声标签学习问题[转]
基于深度神经网络的噪声标签学习-云社区-华为云 (huaweicloud.com)神经网络的成功建立在大量的干净数据和很深的网络模型基础上。但是在现实场景中数据和模型往往不会特别理想，比如数据层面有误标记的情况，像小狗被标注成狼，而且实际的业务场景讲究时效性，神经网络的层数不能特别深。我们尝试不断迭代数据和模型缺陷情况下神经网络的有效训练方法，通过noisy label learning技术，解决网络训练过程中noisy data的问题，该技术已经在团队实际业务场景中落地，通过从损失函数、网络结构、模型正则化、损失函数调整、样本选择、标签纠正等多个模块的优化，不局限于全监督、半监督和自监督学习方法，提升整个模型的鲁棒性1. 损失函数主要是从损失函数去修改，核心思路是当数据整体是干净的时候，传统的交叉熵损失函数学习到少量的负样本，可以提升模型的鲁棒性；当数据噪声比较大时，CE会被噪声数据带跑偏，我们要修改损失函数使其在训练中每个样本的权重都是一样重要的，因此不难想到采用GCE Loss，控制超参数，结合了CE Loss和MAE LossA. Ghosh, H. Kumar, and P. Sastry,“Robust loss functions under label noise for deep neural networks,” in Proc. AAAI, 2017Generalized Cross Entropy Loss for Training Deep Neural Networks with Noisy Labels, NeurlPS 2018另外，还有从KL散度想法借鉴过来的，作者认为在计算熵的时候，原始q, p代表的真实数据分布和预测值在较为干净的数据上没有问题，但是在噪声比较大的数据上，可能q并不能代表真实数据分布，相反的是不是p可以表示真实数据分布，因此提出基于对称的交叉熵损失函数（Symmetric cross entropy ）Y. Wang, X. Ma, Z. Chen, Y. Luo, J. Yi, and J. Bailey, “Symmetric cross entropy for robust learning with noisy labels,” in Proc. ICCV, 2019, pp. 322–3302. 模型结构主要通过借鉴巧妙的网络结构，在模型训练过程中，通过模型对数据进行挑选，选择一批较为干净的数据，逐步提升模型的鲁棒性。首先要介绍的就是coteaching framework，首先是基于两个模型相互挑选数据输入给对方计算loss，传递给对方网络的数据是每个min-batch里面loss最低的一些数据，随着epoch增加，数据量有所变化，另外每一轮epoch结束，会shuffle数据，保证数据不会被永久遗忘How does Disagreement Help Generalization against Label Corruption? ICML 2019Co-teaching: Robust Training of Deep Neural Networks with Extremely Noisy Labels, NeurlPS 2018还有一种思路，是基于attention注意力机制给干净样本和噪声数据进行打分，文章叫做Attention Feature Mixup，在计算最终loss的时候有两部分，一部分是同一个类的每张图和标签计算的交叉熵损失；另外一个损失是数据mixup得到的新的数据x'和标签y'计算的loss3. 正则化主要是通过一些添加正则ticks，防止模型过拟合到噪声数据上，常用的正则方法包含：label smooth、l1、l2、MixUp等4. 样本选择主要是从如何挑选出更好的数据为出发点，一种方法，叫做Area Under the Margin metric (AUM)，是我们去年参加CVPR WebVision 2020（图像识别领域最高级别的比赛，接替ImageNet）取得冠军的方案。该方案是一种在训练过程中一边训练一边筛选数据的方式，具体思想是在一个min-batch利用模型计算每张图片的logits值和其它类中最大的logits求差值当做area，这样多轮epoch求平均，得到每张图的aum值。试验发现如果这个数据是较为干净的数据area的值会比较大，如果是存在mis-label的数据area值比较小，甚至是负值，作者就是通过这个思想将一个类的干净数据和噪声数据分离开的。当然论文在最后也指出，干净数据和噪声数据占比99%的阈值是最优的。Pleiss, Geoff, et al. "Identifying mislabeled data using the area under the margin ranking.“, NeurlPS 2020.另外一篇，数据划分是通过密度聚类的思路，将一个类的数据分成easy dataset、smi-hard dataset和hard dataset，一般噪声数据是较为困难训练的数据，对于每张图分配一个权重，文中建议1.0、0.5和0.5；模型的训练借鉴了课程学习的思路Guo, Sheng, et al. "Curriculumnet: Weakly supervised learning from large-scale web images." Proceedings of the European Conference on Computer Vision (ECCV). 2018.5. 标签纠正标签纠正(label correction)方法思路很简单就是相当于是一个重新打一个伪标签的概念，但是完全放弃原始的标签也过于暴力，ICCV2019的这篇文章在“label correction phase”通过一个pre-trained模型得到随机选择每个类中的几张图采用聚类的方法得到Prototype样本的每个类的聚类中心，对输入图片得到的特征向量和各类聚类中心计算距离，得到图片的伪标签，最后的loss是原始标签计算的交叉熵损失和伪标签计算的伪标签的求和Han, Jiangfan, Ping Luo, and Xiaogang Wang. "Deep self-learning from noisy labels.“, ICCV 2019
原文：A Framework For Contrastive Self-Supervised Learning And Designing A New Approach-2020.09.03论文：A Framework For Contrastive Self-Supervised Learning And Designing A New Approach-2020节选，学习.关于对比学习和自监督学习，PyTorch Lightning 开源了相关的代码实现：SELF-SUPERVISED LEARNING，如：[1] - Augmented Multiscale Deep InfoMax (AMDIM)[2] - Bootstrap Your Own Latent (BYOL)[3] - Data-Efficient Image Recognition with Contrastive Predictive Coding[4] - Moco[5] - SimCLR[6] - SwAV[7] - Exploring Simple Siamese Representation Learning (SimSiam)1. 自监督学习监督学习：给定输入 x 和标签 y.图：左-输入x，右-标签y自监督学习：仅有输入 x. 模型学习从输入的其他部分预测输入的一部分.图：左-输入；右-标签自监督学习问题比较通用，相关的前置任务(pretext task)如：[1] - 预测两个图像块的相对位置 [predicting relative locations of two patches][2] - 拼图 [solving a jigsaw puzzle][3] - 图像上色 [colorizing an image]对比学习极大地缩小了在 ImageNet 上与监督学习方法的差距.2. 对比学习大部分机器学习算法的基本思想是，相似样本聚集，而与其他聚类的样本远离.Learning a Similarity Metric Discriminatively, with Application to Face Verification - 2004动画如：对比学习三个部分：anchor、positive、negeative.正样本对，两个样本相似；负样本对，不相似的第三个样本.但是，自监督学习中，样本的标签是未知的，因此无法确定两张图片是否相似.不过，如果假设每个图片是一个类，那么就可以想办法构建 triplets（正负样本对），也就意味着数据集大小为 N 时，会存在 N 个标签.因此，就可以知道每张图片的标签了，可以采用数据增强生成 triplets.2.1. 特征1 - 数据增强第一种表征对比自监督学习方法的方式是，定义数据增强pipline.数据增强pipline $A(x)$ 对相同的输入采用一序列的随机变换.深度学习中，数据增强旨在构建原始输入的对噪声鲁棒的表示. 例如，即使图片旋转，或去除颜色，或像素抖动，网络模型也应该能识别.而对比学习中，数据增强还有第二个作用，即，用于生成 anchor、positive、negative 样本，以送入编码器及提取特征表示.2.1.1. CPCCPC pipline 除了采用如颜色抖动、灰度化、随机翻转等变换，还进行一种特殊变换，即将图片划分为重叠子图像块.图：CPC 的关键变换.CPC 可以生成很多 positive 和 negative 样本集. 图：从 batch 图片中生成positive、anchor、negative 样本对[如，batchsize=3]2.1.2. AMDIMAMDIM 采用的方法略有不同. 其在标准的数据增强后[如，抖动、翻转等]，再通过对相同图片进行两次数据增强，以生成一张图片的两个版本.这种想法实际上是在论文 Discriminative Unsupervised Feature Learning with Convolutional Neural Networks-2014中提出的. 其是采用 seed 图片生成相同图片的许多版本.2.1.3. SimCLR可见视频：https://www.youtube.com/embed/Mrp2ntS2QxI2.2. 特征2 - 编码器第二种表征对比自监督学习方法的方式是，编码器的选择. 大部分方法采用的是 ResNets 及其衍生网络.2.3. 特征3 - 表征提取第三种表征对比自监督学习方法的方式是，提取特征表示所采用的策略.表征的定义：其是一组独特的特征，使得系统和人类能够理解是什么构成了那个对象，那个对象，而不是不同的对象.A representation is the set of unique characteristics that allow a system (and humans) to understand what makes that object, that object, and not a different one.如图，一种好的特征表示可能是角的数量.What is representation learning in deep learning? - Quora2.3.1. CPCCPC 提出了通过在隐空间（latent space）预测未来（future）的方式学习表征. 其包含两个方面：[1] - 将每张图片看作是时间线，从左上角（过去）到右下角（将来）[2] - 预测并不是在像素层进行的，而是采用编码器的输出（如，隐空间latent space）图：从像素空间到隐空间最后，通过将编码器H的输出作为投影head（论文作者称之为上下文编码器）生成的上下文向量的目标，来构造预测任务，以进行表示提取.图：CPC 表示提取2.3.2. AMDIMAMDIM 的思想是，使用从 CNN 网络中间层提取的特征图的视图间（across views）的表示进行比较.可以将其分解为两部分：单张图像的多视角；CNN的中间层.[1] - AMDIM 的数据增强中，生成同一张图片的两个版本.[2 ]- 每个版本的图片均送入相同的编码器，以每张图片的提取特征图. AMDIM 并未丢弃编码器生成的中间特征图，而是对其进行跨空间尺度（spatial scales）的对比. 如图，CNN中间层的接收野编码了输入的不同尺度的信息，AMDIM 跨CNN中间层输出进行对比. 如图，跨编码器生成的三个特征图进行对比.图：AMDIM 表示提取：AMDIM 采用相同的编码器提取 3 组特征图集合；然后，跨特征图进行对比.2.3.3. SimCLRSimCLR 采用了类似于 AMDIM 的思想，但有2处修改.[1] - 仅使用最后一个特征图;[2] - 将特征图井投影head 处理，再比较两个向量.2.3.4. BYOLBYOL 采用与 AMDIM 相同的思想(但仅采用最后一个特征图)，但有两处改动：[1] - BYOL 采用了两个编码器，而不是一个. 第二个编码器实际上是第一个编码器的副本，以滚动平均的方式更新权重，而不是每次迭代中更新权重.[2] - BYOL 不需要使用负样本. 相反，其依靠滚动权重更新来给训练提供对比信号.2.3.5. Swav将表示提取任务看作为在线聚类（online clustering）的一种，其中，强制保持相同图片不同数据增强的代码之间的一致性. 类似于AMDIM（仅使用最后一个特征图），但其不是直接将向量相互比较，而是根据一组 K 个预计算代码来计算相似度.实际上，这意味着，Swav 会生成 K 个聚类，并且对于每个编码向量，其将于这些聚类进行比较，以学习新的表示.2.4. 特征4 - 相似性度量第四种表征对比自监督学习方法的方式是，所采用的相似度度量方式.上面这些方法均采用的是点积（dot product）或余弦相似性（cosine similarity）.本文作者实验表明，相似性的选择很大程度是不那么重要的.2.5. 特征5 - 损失函数第五种表征对比自监督学习方法的方式是，所采用的损失函数.除了 BYOL，均采用了NCELoss.NCELoss 包含两部分，分子和分母. 分子鼓励相似的向量彼此靠近，分母把所有其他向量推的更远.如果分母不存在，则损失变为一个常数，所学的特征表示也将没有用处.https://youtu.be/_1eKr4rbgRI看到这篇博文最后，只能感叹大厂才玩得起的钞能力.
论文：MoCov1 - Momentum Contrast for Unsupervised Visual Representation Learning - 2019论文：MoCov2 - Improved Baselines with Momentum Contrastive Learning - 2020Github - facebookresearch/mocoMoCo，Momentum Contrast，是一种无监督视觉表示学习算法. MoCo 基于对比学习(contrastive learning) 的角度，将问题转换为字典查询(dictionary look-up)，构建了基于队列的动态字典更新(dynamic dictionary with a queue)和滑动平均编码器(moving-averaged encoder)的算法.1. MoCoMoCo 基于的假设：可以通过覆盖大量负样本的大型字典来学习好的特征，且字典 keys 的编码器尽管在不断的更新，但仍应尽可能保持连续性.对于给定的一个样本x， 选择一个(或一批)正样本y(这里正样本对对于图像上的理解就是x的不同的data augmentation版本). 然后选择一批负样本(对于图像来说，就是除了x之外的图像), 然后设计loss function来将x与正样本之间的距离拉近，负样本之间的距离推开.MoCo 主要有如下特点：1.1. InfoNCE 损失函数[1] - 将对比学习问题转化为字典查询问题损失函数(InfoNCE)形式如：$$ L_q = -log \frac{exp(q \cdot k_+/ \tau)}{\sum_{i=0}^K exp(q \cdot k_i / \tau)} $$另一种形式如，$$ L_{q, k_+, \lbrace k_- \rbrace} = -log \frac{exp(q \cdot k_+/ \tau)}{exp(q \cdot k_+/ \tau) + \sum_{k_-} exp(q \cdot k_- / \tau)} $$其中，$q$ 是查询的特征表示；$k_+$ 是正样本(相似)的特征表示，$\lbrace k_- \rbrace$ 是负样本(不相似)的特征表示；$\tau$ 是超参数.损失函数的求和，是对一个正样本和K个负样本进行的，其类似于 (K+1) Softmax 分类的 log 损失函数，其目标是将$q$ 分类为 $k_+$. 参考 SoftmaxEntropyCross 损失：$$ L_{ce} = -log \frac{exp(x_i)}{\sum_j exp(x_j)} $$或，$$ L(x, class) = -log (\frac{exp(x[class])}{\sum_j exp(x[j])}) $$1.2. 动量更新记 $f_k$ 为 key 编码器；$f_q$ 为 q 查询编码器，$\theta_k$ 和 $\theta_q$ 分别为其参数，MoCo 提出的 $\theta_k$ 动量更新方式如：$$ \theta _k \leftarrow m \theta_k + (1 - m) \theta_q $$其中，只有 $\theta_q$ 是需要 BP 梯度计算的. $m$ 为动量系数，实验证明，相对大的动量(如，m=0.999) 比相对小的动量(如，m=0.9) 的效果好很多.1.3. 将字典作为队列MoCo 的核心是，将字典维护为数据样本的队列. 有利于重用训练过程中的历史 mini-batches 的编码 keys. 基于队列的实现，可以将字典大小和 mini-batch size 解耦，且更大. 队列中的样本是渐进式被替代的. 当前 mini-batch 入队列；最老的 mini-batch 出队列. 字典 keys 一般都所有数据的部分采样子集.1.4. Shuffling BNMoCo 论文所述，在标准 ResNet 的 BN 层，会影响模型的训练. 因为模型可能在试图欺骗任务，并容易地找到loss值比较低的方案.而，MoCo 采用的方式是 shuffling BN. 采用多个 GPU 进行训练，分别对每个 GPU 上的样本进行独立的进行BN. 即，对于 key 编码器 $f_k$，在将其分发到各 GPUs 上时，打乱当前 mini-batch 的采样次序；而查询编码器 $f_q$ 的mini-batch 的采样次序保持不变.1.5. MoCo 实现伪代码2. MoCoV2MoCoV2 的提出源于 SimCLR - A Simple Framework for Contrastive Learning of Visual Representations,SimCLR 是主要是有三方面的提升：[1] - 较大的batchsize(如4k或8k)，能够提供更多的负样本；[2] - 将输出 fc 层 head 网络替换为 MLP head；[3] - 更多的数据增强.而 MoCoV2 发现，MoCoV1 + MLP head + more data aug 也能取得更好的结果，而且还不需要大的训练 batchsize. 如图，从 batch 角度的理解, 基于 menmory queue 的机制，不需要很大的 batchsize，MoCo 也能提供更多的负样本进行对比学习.3. MoCo Pytorch 实现3.1. MoCo 构建moco/builder.pyimport torch import torch.nn as nn class MoCo(nn.Module): """ Build a MoCo model with: a query encoder, a key encoder, and a queue https://arxiv.org/abs/1911.05722 """ def __init__(self, base_encoder, dim=128, K=65536, m=0.999, T=0.07, mlp=False): """ dim: 特征维度feature dimension (default: 128) K: 队列大小queue size; 负样本keys数number of negative keys (default: 65536) m: moco momentum of updating key encoder (default: 0.999) T: softmax temperature (default: 0.07) """ super(MoCo, self).__init__() self.K = K self.m = m self.T = T # 创建编码器，这里是基于 resnet50 # num_classes：fc层输出的维度 self.encoder_q = base_encoder(num_classes=dim) self.encoder_k = base_encoder(num_classes=dim) if mlp: # hack: brute-force replacement; MoCov2 dim_mlp = self.encoder_q.fc.weight.shape[1] self.encoder_q.fc = nn.Sequential(nn.Linear(dim_mlp, dim_mlp), nn.ReLU(), self.encoder_q.fc) self.encoder_k.fc = nn.Sequential(nn.Linear(dim_mlp, dim_mlp), nn.ReLU(), self.encoder_k.fc) for param_q, param_k in zip(self.encoder_q.parameters(), self.encoder_k.parameters()): param_k.data.copy_(param_q.data) #初始化编码器k的参数 param_k.requires_grad = False #编码器k不需要梯度计算 #创建队列 self.register_buffer("queue", torch.randn(dim, K)) self.queue = nn.functional.normalize(self.queue, dim=0) self.register_buffer("queue_ptr", torch.zeros(1, dtype=torch.long)) @torch.no_grad() def _momentum_update_key_encoder(self): """ 动量更新编码器 key 的参数 """ for param_q, param_k in zip(self.encoder_q.parameters(), self.encoder_k.parameters()): param_k.data = param_k.data * self.m + param_q.data * (1. - self.m) @torch.no_grad() def _dequeue_and_enqueue(self, keys): #更新队列前，收集keys keys = concat_all_gather(keys) batch_size = keys.shape[0] ptr = int(self.queue_ptr) assert self.K % batch_size == 0 # for simplicity #在 ptr 处替换 keys，出队列和入队列 self.queue[:, ptr:ptr + batch_size] = keys.T ptr = (ptr + batch_size) % self.K # move pointer self.queue_ptr[0] = ptr @torch.no_grad() def _batch_shuffle_ddp(self, x): """ Batch shuffle, for making use of BatchNorm. *** Only support DistributedDataParallel (DDP) model. *** """ # gather from all gpus batch_size_this = x.shape[0] x_gather = concat_all_gather(x) batch_size_all = x_gather.shape[0] num_gpus = batch_size_all // batch_size_this # random shuffle index idx_shuffle = torch.randperm(batch_size_all).cuda() # broadcast to all gpus torch.distributed.broadcast(idx_shuffle, src=0) # index for restoring idx_unshuffle = torch.argsort(idx_shuffle) # shuffled index for this gpu gpu_idx = torch.distributed.get_rank() idx_this = idx_shuffle.view(num_gpus, -1)[gpu_idx] return x_gather[idx_this], idx_unshuffle @torch.no_grad() def _batch_unshuffle_ddp(self, x, idx_unshuffle): """ Undo batch shuffle. *** Only support DistributedDataParallel (DDP) model. *** """ # gather from all gpus batch_size_this = x.shape[0] x_gather = concat_all_gather(x) batch_size_all = x_gather.shape[0] num_gpus = batch_size_all // batch_size_this # restored index for this gpu gpu_idx = torch.distributed.get_rank() idx_this = idx_unshuffle.view(num_gpus, -1)[gpu_idx] return x_gather[idx_this] def forward(self, im_q, im_k): """ Input: im_q: a batch of query images im_k: a batch of key images Output: logits, targets """ # compute query features，计算查询特征 q = self.encoder_q(im_q) # queries: NxC q = nn.functional.normalize(q, dim=1) # compute key features with torch.no_grad(): # no gradient to keys self._momentum_update_key_encoder() # update the key encoder # shuffle for making use of BN im_k, idx_unshuffle = self._batch_shuffle_ddp(im_k) k = self.encoder_k(im_k) # keys: NxC k = nn.functional.normalize(k, dim=1) # undo shuffle k = self._batch_unshuffle_ddp(k, idx_unshuffle) # compute logits # Einstein sum is more intuitive # positive logits: Nx1，正样本 l_pos = torch.einsum('nc,nc->n', [q, k]).unsqueeze(-1) # negative logits: NxK，队列里都是负样本 l_neg = torch.einsum('nc,ck->nk', [q, self.queue.clone().detach()]) # logits: Nx(1+K) logits = torch.cat([l_pos, l_neg], dim=1) #第0列是正样本 # apply temperature logits /= self.T # labels: positive key indicators labels = torch.zeros(logits.shape[0], dtype=torch.long).cuda() # https://github.com/facebookresearch/moco/issues/24 # dequeue and enqueue self._dequeue_and_enqueue(k) return logits, labels # utils @torch.no_grad() def concat_all_gather(tensor): """ Performs all_gather operation on the provided tensors. *** Warning ***: torch.distributed.all_gather has no gradient. """ tensors_gather = [torch.ones_like(tensor) for _ in range(torch.distributed.get_world_size())] torch.distributed.all_gather(tensors_gather, tensor, async_op=False) output = torch.cat(tensors_gather, dim=0) return output3.2. MoCo 训练coco/loader.pyfrom PIL import ImageFilter import random class TwoCropsTransform: """ Take two random crops of one image as the query and key. """ def __init__(self, base_transform): self.base_transform = base_transform def __call__(self, x): q = self.base_transform(x) k = self.base_transform(x) return [q, k] class GaussianBlur(object): """ Gaussian blur augmentation in SimCLR https://arxiv.org/abs/2002.05709 """ def __init__(self, sigma=[.1, 2.]): self.sigma = sigma def __call__(self, x): sigma = random.uniform(self.sigma[0], self.sigma[1]) x = x.filter(ImageFilter.GaussianBlur(radius=sigma)) return x训练主函数:main_moco.py#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- import argparse import builtins import math import os import random import shutil import time import warnings import torch import torch.nn as nn import torch.nn.parallel import torch.backends.cudnn as cudnn import torch.distributed as dist import torch.optim import torch.multiprocessing as mp import torch.utils.data import torch.utils.data.distributed import torchvision.transforms as transforms import torchvision.datasets as datasets import torchvision.models as models import moco.loader #数据加载与增强辅助函数 import moco.builder #MoCo模型函数 model_names = sorted(name for name in models.__dict__ if name.islower() and not name.startswith("__") and callable(models.__dict__[name])) parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') parser.add_argument('data', metavar='DIR', help='path to dataset') parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50', choices=model_names, help='model architecture: ' + ' | '.join(model_names) + ' (default: resnet50)') parser.add_argument('-j', '--workers', default=32, type=int, metavar='N', help='number of data loading workers (default: 32)') parser.add_argument('--epochs', default=200, type=int, metavar='N', help='number of total epochs to run') parser.add_argument('--start-epoch', default=0, type=int, metavar='N', help='manual epoch number (useful on restarts)') parser.add_argument('-b', '--batch-size', default=256, type=int, metavar='N', help='mini-batch size (default: 256), this is the total ' 'batch size of all GPUs on the current node when ' 'using Data Parallel or Distributed Data Parallel') parser.add_argument('--lr', '--learning-rate', default=0.03, type=float, metavar='LR', help='initial learning rate', dest='lr') parser.add_argument('--schedule', default=[120, 160], nargs='*', type=int, help='learning rate schedule (when to drop lr by 10x)') parser.add_argument('--momentum', default=0.9, type=float, metavar='M', help='momentum of SGD solver') parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float, metavar='W', help='weight decay (default: 1e-4)', dest='weight_decay') parser.add_argument('-p', '--print-freq', default=10, type=int, metavar='N', help='print frequency (default: 10)') parser.add_argument('--resume', default='', type=str, metavar='PATH', help='path to latest checkpoint (default: none)') parser.add_argument('--world-size', default=-1, type=int, help='number of nodes for distributed training') parser.add_argument('--rank', default=-1, type=int, help='node rank for distributed training') parser.add_argument('--dist-url', default='tcp://224.66.41.62:23456', type=str, help='url used to set up distributed training') parser.add_argument('--dist-backend', default='nccl', type=str, help='distributed backend') parser.add_argument('--seed', default=None, type=int, help='seed for initializing training. ') parser.add_argument('--gpu', default=None, type=int, help='GPU id to use.') parser.add_argument('--multiprocessing-distributed', action='store_true', help='Use multi-processing distributed training to launch ' 'N processes per node, which has N GPUs. This is the ' 'fastest way to use PyTorch for either single node or ' 'multi node data parallel training') # moco specific configs: parser.add_argument('--moco-dim', default=128, type=int, help='feature dimension (default: 128)') parser.add_argument('--moco-k', default=65536, type=int, help='queue size; number of negative keys (default: 65536)') parser.add_argument('--moco-m', default=0.999, type=float, help='moco momentum of updating key encoder (default: 0.999)') parser.add_argument('--moco-t', default=0.07, type=float, help='softmax temperature (default: 0.07)') # options for moco v2 parser.add_argument('--mlp', action='store_true', help='use mlp head') parser.add_argument('--aug-plus', action='store_true', help='use moco v2 data augmentation') parser.add_argument('--cos', action='store_true', help='use cosine lr schedule') def main(): args = parser.parse_args() if args.seed is not None: random.seed(args.seed) torch.manual_seed(args.seed) cudnn.deterministic = True warnings.warn('You have chosen to seed training. ' 'This will turn on the CUDNN deterministic setting, ' 'which can slow down your training considerably! ' 'You may see unexpected behavior when restarting ' 'from checkpoints.') if args.gpu is not None: warnings.warn('You have chosen a specific GPU. This will completely ' 'disable data parallelism.') if args.dist_url == "env://" and args.world_size == -1: args.world_size = int(os.environ["WORLD_SIZE"]) args.distributed = args.world_size > 1 or args.multiprocessing_distributed ngpus_per_node = torch.cuda.device_count() if args.multiprocessing_distributed: # Since we have ngpus_per_node processes per node, the total world_size # needs to be adjusted accordingly args.world_size = ngpus_per_node * args.world_size # Use torch.multiprocessing.spawn to launch distributed processes: the # main_worker process function mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) else: # Simply call main_worker function main_worker(args.gpu, ngpus_per_node, args) def main_worker(gpu, ngpus_per_node, args): args.gpu = gpu # suppress printing if not master if args.multiprocessing_distributed and args.gpu != 0: def print_pass(*args): pass builtins.print = print_pass if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) if args.distributed: if args.dist_url == "env://" and args.rank == -1: args.rank = int(os.environ["RANK"]) if args.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) #创建 MoCo 模型 print("=> creating model '{}'".format(args.arch)) model = moco.builder.MoCo( models.__dict__[args.arch], args.moco_dim, args.moco_k, args.moco_m, args.moco_t, args.mlp) print(model) if args.distributed: # For multiprocessing distributed, DistributedDataParallel constructor # should always set the single device scope, otherwise, # DistributedDataParallel will use all available devices. if args.gpu is not None: torch.cuda.set_device(args.gpu) model.cuda(args.gpu) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all # available GPUs if device_ids are not set model = torch.nn.parallel.DistributedDataParallel(model) elif args.gpu is not None: torch.cuda.set_device(args.gpu) model = model.cuda(args.gpu) # comment out the following line for debugging raise NotImplementedError("Only DistributedDataParallel is supported.") else: # AllGather implementation (batch shuffle, queue update, etc.) in # this code only supports DistributedDataParallel. raise NotImplementedError("Only DistributedDataParallel is supported.") #定义损失函数(criterion) criterion = nn.CrossEntropyLoss().cuda(args.gpu)#交叉熵损失函数 #定义优化器 optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) if args.gpu is None: checkpoint = torch.load(args.resume) else: # Map model to be loaded to specified single gpu. loc = 'cuda:{}'.format(args.gpu) checkpoint = torch.load(args.resume, map_location=loc) args.start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True #数据加载 traindir = os.path.join(args.data, 'train') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) if args.aug_plus: # MoCo v2's aug: similar to SimCLR https://arxiv.org/abs/2002.05709 augmentation = [ transforms.RandomResizedCrop(224, scale=(0.2, 1.)), transforms.RandomApply([ transforms.ColorJitter(0.4, 0.4, 0.4, 0.1) # not strengthened ], p=0.8), transforms.RandomGrayscale(p=0.2), transforms.RandomApply([moco.loader.GaussianBlur([.1, 2.])], p=0.5), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize ] else: # MoCo v1's aug: the same as InstDisc https://arxiv.org/abs/1805.01978 augmentation = [ transforms.RandomResizedCrop(224, scale=(0.2, 1.)), transforms.RandomGrayscale(p=0.2), transforms.ColorJitter(0.4, 0.4, 0.4, 0.4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize ] #数据集加载，用的是 TwoCropsTransform train_dataset = datasets.ImageFolder( traindir, moco.loader.TwoCropsTransform(transforms.Compose(augmentation))) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) else: train_sampler = None train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler, drop_last=True) for epoch in range(args.start_epoch, args.epochs): if args.distributed: #用于 shuffling. 否则，每次处理都是相同的随机种子 train_sampler.set_epoch(epoch) #学习率调整 adjust_learning_rate(optimizer, epoch, args) # train for one epoch train(train_loader, model, criterion, optimizer, epoch, args) if not args.multiprocessing_distributed or (args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer' : optimizer.state_dict(), }, is_best=False, filename='checkpoint_{:04d}.pth.tar'.format(epoch)) def train(train_loader, model, criterion, optimizer, epoch, args): batch_time = AverageMeter('Time', ':6.3f') data_time = AverageMeter('Data', ':6.3f') losses = AverageMeter('Loss', ':.4e') top1 = AverageMeter('Acc@1', ':6.2f') top5 = AverageMeter('Acc@5', ':6.2f') progress = ProgressMeter( len(train_loader), [batch_time, data_time, losses, top1, top5], prefix="Epoch: [{}]".format(epoch)) # switch to train mode model.train() end = time.time() for i, (images, _) in enumerate(train_loader): # measure data loading time data_time.update(time.time() - end) if args.gpu is not None: images[0] = images[0].cuda(args.gpu, non_blocking=True) images[1] = images[1].cuda(args.gpu, non_blocking=True) # compute output,对比学习，输入是两张图片 output, target = model(im_q=images[0], im_k=images[1]) loss = criterion(output, target) # acc1/acc5 are (K+1)-way contrast classifier accuracy # measure accuracy and record loss acc1, acc5 = accuracy(output, target, topk=(1, 5)) losses.update(loss.item(), images[0].size(0)) top1.update(acc1[0], images[0].size(0)) top5.update(acc5[0], images[0].size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % args.print_freq == 0: progress.display(i) def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): torch.save(state, filename) if is_best: shutil.copyfile(filename, 'model_best.pth.tar') class AverageMeter(object): """Computes and stores the average and current value""" def __init__(self, name, fmt=':f'): self.name = name self.fmt = fmt self.reset() def reset(self): self.val = 0 self.avg = 0 self.sum = 0 self.count = 0 def update(self, val, n=1): self.val = val self.sum += val * n self.count += n self.avg = self.sum / self.count def __str__(self): fmtstr = '{name} {val' + self.fmt + '} ({avg' + self.fmt + '})' return fmtstr.format(**self.__dict__) class ProgressMeter(object): def __init__(self, num_batches, meters, prefix=""): self.batch_fmtstr = self._get_batch_fmtstr(num_batches) self.meters = meters self.prefix = prefix def display(self, batch): entries = [self.prefix + self.batch_fmtstr.format(batch)] entries += [str(meter) for meter in self.meters] print('\t'.join(entries)) def _get_batch_fmtstr(self, num_batches): num_digits = len(str(num_batches // 1)) fmt = '{:' + str(num_digits) + 'd}' return '[' + fmt + '/' + fmt.format(num_batches) + ']' #学习率调整策略 def adjust_learning_rate(optimizer, epoch, args): """ Decay the learning rate based on schedule """ lr = args.lr if args.cos: # cosine lr schedule # cos lr lr *= 0.5 * (1. + math.cos(math.pi * epoch / args.epochs)) else: # stepwise lr schedule for milestone in args.schedule: lr *= 0.1 if epoch >= milestone else 1. for param_group in optimizer.param_groups: param_group['lr'] = lr def accuracy(output, target, topk=(1,)): """ Computes the accuracy over the k top predictions for the specified values of k """ with torch.no_grad(): maxk = max(topk) batch_size = target.size(0) _, pred = output.topk(maxk, 1, True, True) pred = pred.t() correct = pred.eq(target.view(1, -1).expand_as(pred)) res = [] for k in topk: correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) res.append(correct_k.mul_(100.0 / batch_size)) return res if __name__ == '__main__': main()
Github - AutoAugment论文：AutoAugment - Learning Augmentation Policies from Data - 2018Google AI Blogpost - improving-deep-learning-performance图：ImageNet 数据集上成功的策略. 可以看出，其大部分策略只要是颜色变换.1. autoaugment.pyautoaugment.pyfrom PIL import Image, ImageEnhance, ImageOps import numpy as np import random class ImageNetPolicy(object): """ Randomly choose one of the best 24 Sub-policies on ImageNet. Example: >>> policy = ImageNetPolicy() >>> transformed = policy(image) Example as a PyTorch Transform: >>> transform=transforms.Compose([ >>> transforms.Resize(256), >>> ImageNetPolicy(), >>> transforms.ToTensor()]) """ def __init__(self, fillcolor=(128, 128, 128)): #数据增强策略列表 self.policies = [ SubPolicy(0.4, "posterize", 8, 0.6, "rotate", 9, fillcolor), SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), SubPolicy(0.8, "equalize", 8, 0.6, "equalize", 3, fillcolor), SubPolicy(0.6, "posterize", 7, 0.6, "posterize", 6, fillcolor), SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), SubPolicy(0.4, "equalize", 4, 0.8, "rotate", 8, fillcolor), SubPolicy(0.6, "solarize", 3, 0.6, "equalize", 7, fillcolor), SubPolicy(0.8, "posterize", 5, 1.0, "equalize", 2, fillcolor), SubPolicy(0.2, "rotate", 3, 0.6, "solarize", 8, fillcolor), SubPolicy(0.6, "equalize", 8, 0.4, "posterize", 6, fillcolor), SubPolicy(0.8, "rotate", 8, 0.4, "color", 0, fillcolor), SubPolicy(0.4, "rotate", 9, 0.6, "equalize", 2, fillcolor), SubPolicy(0.0, "equalize", 7, 0.8, "equalize", 8, fillcolor), SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor), SubPolicy(0.8, "rotate", 8, 1.0, "color", 2, fillcolor), SubPolicy(0.8, "color", 8, 0.8, "solarize", 7, fillcolor), SubPolicy(0.4, "sharpness", 7, 0.6, "invert", 8, fillcolor), SubPolicy(0.6, "shearX", 5, 1.0, "equalize", 9, fillcolor), SubPolicy(0.4, "color", 0, 0.6, "equalize", 3, fillcolor), SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor), SubPolicy(0.8, "equalize", 8, 0.6, "equalize", 3, fillcolor) ] def __call__(self, img): #随机选择策略 policy_idx = random.randint(0, len(self.policies) - 1) return self.policies[policy_idx](img) def __repr__(self): return "AutoAugment ImageNet Policy" class CIFAR10Policy(object): """ Randomly choose one of the best 25 Sub-policies on CIFAR10. Example: >>> policy = CIFAR10Policy() >>> transformed = policy(image) Example as a PyTorch Transform: >>> transform=transforms.Compose([ >>> transforms.Resize(256), >>> CIFAR10Policy(), >>> transforms.ToTensor()]) """ def __init__(self, fillcolor=(128, 128, 128)): self.policies = [ SubPolicy(0.1, "invert", 7, 0.2, "contrast", 6, fillcolor), SubPolicy(0.7, "rotate", 2, 0.3, "translateX", 9, fillcolor), SubPolicy(0.8, "sharpness", 1, 0.9, "sharpness", 3, fillcolor), SubPolicy(0.5, "shearY", 8, 0.7, "translateY", 9, fillcolor), SubPolicy(0.5, "autocontrast", 8, 0.9, "equalize", 2, fillcolor), SubPolicy(0.2, "shearY", 7, 0.3, "posterize", 7, fillcolor), SubPolicy(0.4, "color", 3, 0.6, "brightness", 7, fillcolor), SubPolicy(0.3, "sharpness", 9, 0.7, "brightness", 9, fillcolor), SubPolicy(0.6, "equalize", 5, 0.5, "equalize", 1, fillcolor), SubPolicy(0.6, "contrast", 7, 0.6, "sharpness", 5, fillcolor), SubPolicy(0.7, "color", 7, 0.5, "translateX", 8, fillcolor), SubPolicy(0.3, "equalize", 7, 0.4, "autocontrast", 8, fillcolor), SubPolicy(0.4, "translateY", 3, 0.2, "sharpness", 6, fillcolor), SubPolicy(0.9, "brightness", 6, 0.2, "color", 8, fillcolor), SubPolicy(0.5, "solarize", 2, 0.0, "invert", 3, fillcolor), SubPolicy(0.2, "equalize", 0, 0.6, "autocontrast", 0, fillcolor), SubPolicy(0.2, "equalize", 8, 0.6, "equalize", 4, fillcolor), SubPolicy(0.9, "color", 9, 0.6, "equalize", 6, fillcolor), SubPolicy(0.8, "autocontrast", 4, 0.2, "solarize", 8, fillcolor), SubPolicy(0.1, "brightness", 3, 0.7, "color", 0, fillcolor), SubPolicy(0.4, "solarize", 5, 0.9, "autocontrast", 3, fillcolor), SubPolicy(0.9, "translateY", 9, 0.7, "translateY", 9, fillcolor), SubPolicy(0.9, "autocontrast", 2, 0.8, "solarize", 3, fillcolor), SubPolicy(0.8, "equalize", 8, 0.1, "invert", 3, fillcolor), SubPolicy(0.7, "translateY", 9, 0.9, "autocontrast", 1, fillcolor) ] def __call__(self, img): policy_idx = random.randint(0, len(self.policies) - 1) return self.policies[policy_idx](img) def __repr__(self): return "AutoAugment CIFAR10 Policy" class SVHNPolicy(object): """ Randomly choose one of the best 25 Sub-policies on SVHN. Example: >>> policy = SVHNPolicy() >>> transformed = policy(image) Example as a PyTorch Transform: >>> transform=transforms.Compose([ >>> transforms.Resize(256), >>> SVHNPolicy(), >>> transforms.ToTensor()]) """ def __init__(self, fillcolor=(128, 128, 128)): self.policies = [ SubPolicy(0.9, "shearX", 4, 0.2, "invert", 3, fillcolor), SubPolicy(0.9, "shearY", 8, 0.7, "invert", 5, fillcolor), SubPolicy(0.6, "equalize", 5, 0.6, "solarize", 6, fillcolor), SubPolicy(0.9, "invert", 3, 0.6, "equalize", 3, fillcolor), SubPolicy(0.6, "equalize", 1, 0.9, "rotate", 3, fillcolor), SubPolicy(0.9, "shearX", 4, 0.8, "autocontrast", 3, fillcolor), SubPolicy(0.9, "shearY", 8, 0.4, "invert", 5, fillcolor), SubPolicy(0.9, "shearY", 5, 0.2, "solarize", 6, fillcolor), SubPolicy(0.9, "invert", 6, 0.8, "autocontrast", 1, fillcolor), SubPolicy(0.6, "equalize", 3, 0.9, "rotate", 3, fillcolor), SubPolicy(0.9, "shearX", 4, 0.3, "solarize", 3, fillcolor), SubPolicy(0.8, "shearY", 8, 0.7, "invert", 4, fillcolor), SubPolicy(0.9, "equalize", 5, 0.6, "translateY", 6, fillcolor), SubPolicy(0.9, "invert", 4, 0.6, "equalize", 7, fillcolor), SubPolicy(0.3, "contrast", 3, 0.8, "rotate", 4, fillcolor), SubPolicy(0.8, "invert", 5, 0.0, "translateY", 2, fillcolor), SubPolicy(0.7, "shearY", 6, 0.4, "solarize", 8, fillcolor), SubPolicy(0.6, "invert", 4, 0.8, "rotate", 4, fillcolor), SubPolicy(0.3, "shearY", 7, 0.9, "translateX", 3, fillcolor), SubPolicy(0.1, "shearX", 6, 0.6, "invert", 5, fillcolor), SubPolicy(0.7, "solarize", 2, 0.6, "translateY", 7, fillcolor), SubPolicy(0.8, "shearY", 4, 0.8, "invert", 8, fillcolor), SubPolicy(0.7, "shearX", 9, 0.8, "translateY", 3, fillcolor), SubPolicy(0.8, "shearY", 5, 0.7, "autocontrast", 3, fillcolor), SubPolicy(0.7, "shearX", 2, 0.1, "invert", 5, fillcolor) ] def __call__(self, img): policy_idx = random.randint(0, len(self.policies) - 1) return self.policies[policy_idx](img) def __repr__(self): return "AutoAugment SVHN Policy" class SubPolicy(object): def __init__(self, p1, operation1, magnitude_idx1, p2, operation2, magnitude_idx2, fillcolor=(128, 128, 128)): ranges = { "shearX": np.linspace(0, 0.3, 10), "shearY": np.linspace(0, 0.3, 10), "translateX": np.linspace(0, 150 / 331, 10), "translateY": np.linspace(0, 150 / 331, 10), "rotate": np.linspace(0, 30, 10), "color": np.linspace(0.0, 0.9, 10), "posterize": np.round(np.linspace(8, 4, 10), 0).astype(np.int), "solarize": np.linspace(256, 0, 10), "contrast": np.linspace(0.0, 0.9, 10), "sharpness": np.linspace(0.0, 0.9, 10), "brightness": np.linspace(0.0, 0.9, 10), "autocontrast": [0] * 10, "equalize": [0] * 10, "invert": [0] * 10 } # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand def rotate_with_fill(img, magnitude): rot = img.convert("RGBA").rotate(magnitude) return Image.composite(rot, Image.new("RGBA", rot.size, (128,) * 4), rot).convert(img.mode) func = { "shearX": lambda img, magnitude: img.transform( img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0), Image.BICUBIC, fillcolor=fillcolor), "shearY": lambda img, magnitude: img.transform( img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0), Image.BICUBIC, fillcolor=fillcolor), "translateX": lambda img, magnitude: img.transform( img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0), fillcolor=fillcolor), "translateY": lambda img, magnitude: img.transform( img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])), fillcolor=fillcolor), "rotate": lambda img, magnitude: rotate_with_fill(img, magnitude), "color": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])), "posterize": lambda img, magnitude: ImageOps.posterize(img, magnitude), "solarize": lambda img, magnitude: ImageOps.solarize(img, magnitude), "contrast": lambda img, magnitude: ImageEnhance.Contrast(img).enhance( 1 + magnitude * random.choice([-1, 1])), "sharpness": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance( 1 + magnitude * random.choice([-1, 1])), "brightness": lambda img, magnitude: ImageEnhance.Brightness(img).enhance( 1 + magnitude * random.choice([-1, 1])), "autocontrast": lambda img, magnitude: ImageOps.autocontrast(img), "equalize": lambda img, magnitude: ImageOps.equalize(img), "invert": lambda img, magnitude: ImageOps.invert(img) } self.p1 = p1 self.operation1 = func[operation1] self.magnitude1 = ranges[operation1][magnitude_idx1] self.p2 = p2 self.operation2 = func[operation2] self.magnitude2 = ranges[operation2][magnitude_idx2] def __call__(self, img): if random.random() < self.p1: img = self.operation1(img, self.magnitude1) if random.random() < self.p2: img = self.operation2(img, self.magnitude2) return img2. AutoAugment PIL 用例from autoaugment import ImageNetPolicy image = PIL.Image.open(path) policy = ImageNetPolicy() transformed = policy(image)3. PyTorch ImageNet 用例from autoaugment import ImageNetPolicy data = ImageFolder(rootdir, transform=transforms.Compose( [transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), ImageNetPolicy(), transforms.ToTensor(), transforms.Normalize(...)])) loader = DataLoader(data, ...)
EfficientNet 是一种新的模型缩放方法，准确率比之前最好的Gpipe提高了0.1%，但是模型更小更快，参数的数量和FLOPS都大大减少，效率提升了10倍.1. EfficientNet 简述将 EfficientNet 划分为 base model 和 building block 两部分来分述. 众所周知的，经典 ResNet 模型的 building block 是恒等映射和卷积(identity and convolution block).Efficient 的主要 building block 是 MBConv(mobile inverted bottleneck)，其是在 MobileNetV2 中首先被提出的. 与展开的网络层相比，在 bottlenecks 间直接采用 shortcuts 连接的通道数量会少很对; 结合 depthwise separable convolution，相比较与传统网络层，能够有效的减少计算量，缩减因子几乎是 k^2. 其中，k 表示 kernel size，指定了 2D 卷积滑窗的 height 和 width.By using shortcuts directly between the bottlenecks which connects a much fewer number of channels compared to expansion layers, combined with depthwise separable convolution which effectively reduces computation by almost a factor of k^2, compared to traditional layers. Where k stands for the kernel size, specifying the height and width of the 2D convolution window.EfficientNet 还增加了 squeeze-and-excitation(SE) optimization，其有助于进一步提升性能.另外，EfficientNet 的另一个优势在于，通过仔细的平衡网络深度(depth)、宽度(width) 和分辨率(resolution)，以便更有效的扩展，获得更好的性能.如图，从最小的 EfficientNet B0 到最大的 B7，精度在稳步增加，同时模型参数保持相对小的大小.相比于 ImageNet 同样精度的其它模型，EfficientNet 参数量少很多. 例如，ResNet50 模型在 Keras 实现中参数量共 23534592，即使其精度不如最小的 EfficientNet B0，B0 的参数两仅有 5330564.2. PyTorch - EfficientNetGithub - lukemelas/EfficientNet-PyTorch2.1. 安装Pip 安装：pip install efficientnet_pytorch或源码安装：git clone lukemelas/EfficientNet-PyTorch cd EfficientNet-Pytorch pip install -e .2.2. 加载EfficientNet 的加载：from efficientnet_pytorch import EfficientNet # 加载网络结构，无预训练权重参数 model = EfficientNet.from_name('efficientnet-b0') print(model) # 加载网络结构，带预训练权重参数 model = EfficientNet.from_pretrained('efficientnet-b0') print(model)提供的预训练权重的精度：Name# ParamsTop-1 Acc.Pretrained?efficientnet-b05.3M76.3✓efficientnet-b17.8M78.8✓efficientnet-b29.2M79.8✓efficientnet-b312M81.1✓efficientnet-b419M82.6✓efficientnet-b530M83.3✓efficientnet-b643M84.0✓efficientnet-b766M84.4✓2.3. 修改输出层修改网络输出层：原： (_conv_head): Conv2dStaticSamePadding( 320, 1280, kernel_size=(1, 1), stride=(1, 1), bias=False (static_padding): Identity() ) (_bn1): BatchNorm2d(1280, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True) (_avg_pooling): AdaptiveAvgPool2d(output_size=1) (_dropout): Dropout(p=0.2, inplace=False) (_fc): Linear(in_features=1280, out_features=1000, bias=True) (_swish): MemoryEfficientSwish() )修改：from torch import nn from efficientnet_pytorch import EfficientNet model = EfficientNet.from_pretrained('efficientnet-b0') in_features = model._fc.in_features #或 # model._fc.out_features = 20 model._fc = nn.Linear(in_features=in_features, out_features=20, bias=True) print(model)修改后： (_conv_head): Conv2dStaticSamePadding( 320, 1280, kernel_size=(1, 1), stride=(1, 1), bias=False (static_padding): Identity() ) (_bn1): BatchNorm2d(1280, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True) (_avg_pooling): AdaptiveAvgPool2d(output_size=1) (_dropout): Dropout(p=0.2, inplace=False) (_fc): Linear(in_features=1280, out_features=20, bias=True) (_swish): MemoryEfficientSwish() )2.4. 预测#!/usr/bin/python3 #!--*-- coding:utf-8 --*-- import json from PIL import Image import torch from torchvision import transforms from efficientnet_pytorch import EfficientNet model = EfficientNet.from_pretrained('efficientnet-b0') # 图片处理 tfms = transforms.Compose([transforms.Resize(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),]) # img = tfms(Image.open('test.jpg')).unsqueeze(0) print(img.shape) # torch.Size([1, 3, 224, 224]) # 加载 ImageNet class names labels_map = json.load(open('labels_map.txt')) labels_map = [labels_map[str(i)] for i in range(1000)] # 分类 model.eval() with torch.no_grad(): outputs = model(img) # 预测结果 print('-----') for idx in torch.topk(outputs, k=5).indices.squeeze(0).tolist(): prob = torch.softmax(outputs, dim=1)[0, idx].item() print('{label:<75} ({p:.2f}%)'.format(label=labels_map[idx], p=prob*100))2.5. 特征提取from efficientnet_pytorch import EfficientNet model = EfficientNet.from_pretrained('efficientnet-b0') # 图片处理 tfms = transforms.Compose([transforms.Resize(224), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),]) # img = tfms(Image.open('test.jpg')).unsqueeze(0) print(img.shape) # torch.Size([1, 3, 224, 224]) features = model.extract_features(img) print(features.shape) # torch.Size([1, 1280, 7, 7])2.6. 微调(multi-label 场景)import sys import time from PIL import Image import torchimport torch.nn as nn import torch.optim as optim from torch.utils import data from torch.autograd import Variable import torchvision import torchvision.transforms as transforms from efficientnet_pytorch import EfficientNet # DataLoader class Dataset(data.Dataset): def __init__(self,csv_path,images_path,transform=None): #Read The CSV and create the dataframe self.train_set=pd.read_csv(csv_path) self.train_path=images_path #Images Path self.transform=transform # Augmentation Transforms def __len__(self): return len(self.train_set) def __getitem__(self,idx): file_name=self.train_set.iloc[idx][0]+'.png' label=self.train_set.iloc[idx][1] img=Image.open(os.path.join(self.train_path,file_name)) #Loading Image if self.transform is not None: img=self.transform(img) return img,label # transform_train = transforms.Compose([ transforms.Resize((224,224)), transforms.RandomApply([ torchvision.transforms.RandomRotation(10), transforms.RandomHorizontalFlip()], 0.7), transforms.ToTensor()]) # training_set=Dataset('/path/to/train.csv', '/path/to/train_images/', transform=transform_train) # params = {'batch_size': 16, 'shuffle': True } training_generator=data.DataLoader(training_set,**params) # use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") print(device) outputfolder='/path/to/train_outputs/' if(not os.path.exists(outputfolder)): os.mkdir(outputfolder) # Model model = EfficientNet.from_pretrained('efficientnet-b0', num_classes=20) model.to(device) print(summary(model, input_size=(3, 512, 512))) criterion = nn.CrossEntropyLoss() learning_rate=1e-3 lr_decay=0.99 optimizer = optim.Adam(model.parameters(), lr=learning_rate) # 模型训练 epochs = 100 history_accuracy=[] history_loss=[] for epoch in range(epochs): running_loss = 0.0 correct=0 total=0 class_correct = list(0. for _ in classes) class_total = list(0. for _ in classes) for i, data in enumerate(training_generator, 0): inputs, labels = data t0 = time() inputs, labels = inputs.to(device), labels.to(device) optimizer.zero_grad() outputs = model(inputs) loss = criterion(outputs, torch.max(labels, 1)[1]) _, predicted = torch.max(outputs, 1) _, labels = torch.max(labels, 1) c = (predicted == labels.data).squeeze() correct += (predicted == labels).sum().item() total += labels.size(0) accuracy = float(correct) / float(total) history_accuracy.append(accuracy) history_loss.append(loss) loss.backward() optimizer.step() for j in range(labels.size(0)): label = labels[j] class_correct[label] += c[j].item() class_total[label] += 1 running_loss += loss.item() print( "Epoch : ",epoch+1," Batch : ", i+1," Loss : ",running_loss/(i+1)," Accuracy : ",accuracy,"Time ",round(time()-t0, 2),"s" ) for k in range(len(classes)): if(class_total[k]!=0): print('Accuracy of %5s : %2d %%' % (classes[k], 100 * class_correct[k] / class_total[k])) print('[%d epoch] Accuracy of the network on the Training images: %d %%' % (epoch+1, 100 * correct / total)) if epoch%10==0 or epoch==0: torch.save(model.state_dict(), os.path.join(outputfolder,str(epoch+1)+'_'+str(round(accuracy, 4))+'.pth')) # 模型保存 torch.save(model.state_dict(), os.path.join(outputfolder,'Last_epoch'+str(round(accuracy, 4))+'.pth')) # 可视化训练曲线 plt.plot(history_accuracy) plt.plot(history_loss) # 模型预测 model.load_state_dict(torch.load(os.path.join(outputfolder, '/50_0.9246.pth'))) model.eval() # test_transforms = transforms.Compose([transforms.Resize(512), transforms.ToTensor(), ]) def predict_image(image): image_tensor = test_transforms(image) image_tensor = image_tensor.unsqueeze_(0) input = Variable(image_tensor) input = input.to(device) output = model(input) index = output.data.cpu().numpy().argmax() return index # img=Image.open('test.jpg') prediction=predict_image(img)3. Keras - EfficientNetGithub - qubvel/efficientnet3.1. 安装依赖项：Keras >= 2.2.0 / TensorFlow >= 1.12.0keras_applications >= 1.0.7scikit-imagePip 安装：# stable pip install -U efficientnet # latest - with keras and tf.keras support pip install -U --pre efficientnet # source pip install -U git+https://github.com/qubvel/efficientnet3.2. 加载模型初始化：# models can be build with Keras or Tensorflow frameworks # use keras and tfkeras modules respectively # efficientnet.keras / efficientnet.tfkeras import efficientnet.keras as efn model = efn.EfficientNetB0(weights='imagenet') # or weights='noisy-student'加载预训练权重：# model use some custom objects, so before loading saved model # import module your network was build with # e.g. import efficientnet.keras / import efficientnet.tfkeras import efficientnet.tfkeras from tensorflow.keras.models import load_model model = load_model('path/to/model.h5')提供的云训练权重参数的精度：Architecture@top1* Imagenet@top1* Noisy-StudentEfficientNetB00.7720.788EfficientNetB10.7910.815EfficientNetB20.8020.824EfficientNetB30.8160.841EfficientNetB40.8300.853EfficientNetB50.8370.861EfficientNetB60.8410.864EfficientNetB70.8440.8693.3. 预测import os import sys import numpy as np from skimage.io import imread import matplotlib.pyplot as plt from keras.applications.imagenet_utils import decode_predictions from efficientnet.keras import EfficientNetB0 from efficientnet.keras import center_crop_and_resize, preprocess_input ## 或使用 tensorflow.keras: # from efficientnet.tfkeras import EfficientNetB0 # from efficientnet.tfkeras import center_crop_and_resize, preprocess_input # 测试图片 image = imread('test.jpg') # 加载预训练模型 model = EfficientNetB0(weights='imagenet') # 输入处理 image_size = model.input_shape[1] x = center_crop_and_resize(image, image_size=image_size) x = preprocess_input(x) x = np.expand_dims(x, 0) # 预测及解码 y = model.predict(x) decode_predictions(y)3.4. 微调#!/usr/bin/python3 #!--*-- coding: utf-8 --*-- from efficientnet.tfkeras import EfficientNetB0 from efficientnet.tfkeras import center_crop_and_resize, preprocess_input from tensorflow.keras import models from tensorflow.keras import layers from tensorflow.keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.utils import plot_model # 加载预训练模型，作为 conv base model conv_base = EfficientNetB0(weights="imagenet", include_top=False, input_shape=input_shape) # 构建微调模型 dropout_rate = 0.2 model = models.Sequential() model.add(conv_base) model.add(layers.GlobalMaxPooling2D(name="gap")) # model.add(layers.Flatten(name="flatten")) if dropout_rate > 0: model.add(layers.Dropout(dropout_rate, name="dropout_out")) # model.add(layers.Dense(256, activation='relu', name="fc1")) model.add(layers.Dense(20, activation="softmax", name="fc_out")) # 冻结 conv_base 模型的卷积权重 conv_base.trainable = False # 或 #conv_base.trainable = True #set_trainable = False #for layer in conv_base.layers: # if layer.name == 'multiply_16': # set_trainable = True # if set_trainable: # layer.trainable = True # else: # layer.trainable = False # 训练数据处理 train_datagen = ImageDataGenerator( rescale=1.0 / 255, rotation_range=40, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode="nearest", ) # 验证数据处理 test_datagen = ImageDataGenerator(rescale=1.0 / 255) train_generator = train_datagen.flow_from_directory( train_dir, # All images will be resized to target height and width. target_size=(height, width), batch_size=batch_size, # Since we use categorical_crossentropy loss, we need categorical labels class_mode="categorical", ) validation_generator = test_datagen.flow_from_directory( validation_dir, target_size=(height, width), batch_size=batch_size, class_mode="categorical", ) # model.compile( loss="categorical_crossentropy", optimizer=optimizers.RMSprop(lr=2e-5), metrics=["acc"], ) history = model.fit_generator( train_generator, steps_per_epoch=NUM_TRAIN // batch_size, epochs=epochs, validation_data=validation_generator, validation_steps=NUM_TEST // batch_size, verbose=1, use_multiprocessing=True, workers=4, ) # plot_model(conv_base, to_file='conv_base.png', show_shapes=True)4. 参考[1] - pytorch笔记：Efficientnet微调 - 知乎[2] - How to do Transfer learning with Efficientnet[3] - Keras Applications[4] - 谷歌EfficientNet缩放模型，PyTorch实现出炉，登上GitHub热榜丨Demo可用[5] - Github - kartikdutt18/Kaggle-ATPOS-Efficient_Net[6] - Kaggle - Efficient_Net
图片标注-基于PyTorch的多标签图像分类[译]
原文：Multi-Label Image Classification with PyTorch: Image Tagging - 2020.05.03作者：Victor Bebnev (Xperience.AI)实现：Github - PyTorch-Multi-Label-Image-Classification-Image-Tagging在 图像标注-基于PyTorch的多输出图像分类 中，主要是针对每张图片输出固定数量标签的场景(mult-outputs).1. 多标签分类首先，明确下多标签分类(multi-label classification) 的定义，以及其与多类别分类(multi-class classification) 的区别.根据 scikit-learn 中所述，multi-label 分类是对每个样本分配一组目标标签集；而，multi-class 分类是假设每个样本仅有目标标签集中的一个标签. 此外，multi-label 分类中，每个样本的所有标签不是互斥的(mutually exclusive).在 Multi-Label Image Classification with PyTorch 中，是 multi-output 的分类问题，其处理的也是每个样本多个标签的分类问题. multi-output 分类问题，其往往预测的是每个样本的固定数量的标签；且，理论上可以由相应输出标签数量的独立分类器来替代. 而，multi-label 分类问题往往需要预测非固定数量的标签.multi-output 分类问题必须知道有几个独立的问题，每个问题有且仅有一个可能的答案.为了总结不同分类任务间(multi-class、multi-output、multi-label)的差异性，以下图为例说明. 图-标签可以是：portrait, woman, smiling, brown hair, wavy hairTypePredicted labelsPossible labelsMulti-class classificationsmiling[neutral, smiling, sad]Multi-output classificationwoman, smiling, brown hair[man, woman, child] [neutral, smiling, sad] [brown hair, red hair, blond hair, black hair]Multi-label classificationportrait, woman, smiling, brown hair, wavy hair[portrait, nature, landscape, selfie, man, woman, child, neutral emotion, smiling, sad, brown hair, red hair, blond hair, black hair]在现实生活中，比如 Instagram 标签，用户往往是从标签池里选择某些标签对图片进行标注(为了举例说明，这里假设标签池是固定的). 这些标签往往不是互斥的，它们表征的是图片里所刻画的特征，如“woman”, “wavy hair”, “smiling girl”, “cool shirt”；或者更高层次的语义，如 “portrait”, “happiness”, “fun”.下面以 NUS-WIDE 数据集为例进行说明.2. 数据集这里采用 NUS-WIDE 数据集中的一部分. 该数据集每张图片被标注了多个标签. 其图片收集自 Flickr，包含了超过 1000 个不同标签，最终被收窄到 81 个. 例如，2.1. 数据集划分NUS-WIDE 数据集共包含约 170K 个样本，其标签是高度不均衡的. 例如，对于某些标签，如 sky 和 clouds，数量量约 61000 和 45000；而对于其它标签，如 map 和 earthquake，数量仅为 50.鉴于这里的主要目的是阐述如何处理 multi-label 分类问题，所以将问题进行简化. 仅保留 27 个最频繁出现的标签，使得每个标签具有超过 100 个样本. 此外，为了快速模型训练，仅使用数据集中的前 6000 个样本. 并按照 5:1 来划分为 train 和 val 数据集. 最终得到训练数据集为 5000 张图片，测试数据集为 1000 张图片.图- NUS-WIDE 数据集的标签分布.2.2. 数据处理实现每条标注数据格式类似于：[ 'samples': [ {'image_name': '51353_2739084573_16e9be31f5_m.jpg' , 'image_labels': ['clouds', 'sky']} ... ] 'labels' : ['house', 'birds', 'sun', 'valley', 'nighttime', 'boats', ...] ]代码实现：import os import time import numpy as np from PIL import Image from torch.utils.data.dataset import Dataset from tqdm import tqdm from torchvision import transforms from torchvision import models import torch from torch.utils.tensorboard import SummaryWriter from sklearn.metrics import precision_score, recall_score, f1_score from torch import nn from torch.utils.data.dataloader import DataLoader from matplotlib import pyplot as plt from numpy import printoptions import requests import tarfile import random import json from shutil import copyfile # For reproducible torch.manual_seed(2020) torch.cuda.manual_seed(2020) np.random.seed(2020) random.seed(2020) torch.backends.cudnn.deterministic = True # 数据集下载 img_folder = 'images' if not os.path.exists(img_folder): def download_file_from_google_drive(id, destination): def get_confirm_token(response): for key, value in response.cookies.items(): if key.startswith('download_warning'): return value return None def save_response_content(response, destination): CHUNK_SIZE = 32768 with open(destination, "wb") as f: for chunk in tqdm(response.iter_content(CHUNK_SIZE), desc='Downloading'): if chunk: # filter out keep-alive new chunks f.write(chunk) URL = "https://docs.google.com/uc?export=download" session = requests.Session() response = session.get(URL, params={'id': id}, stream=True) token = get_confirm_token(response) if token: params = {'id': id, 'confirm': token} response = session.get(URL, params=params, stream=True) save_response_content(response, destination) # file_id = '0B7IzDz-4yH_HMFdiSE44R1lselE' path_to_tar_file = str(time.time()) + '.tar.gz' download_file_from_google_drive(file_id, path_to_tar_file) print('Extraction') with tarfile.open(path_to_tar_file) as tar_ref: tar_ref.extractall(os.path.dirname(img_folder)) os.remove(path_to_tar_file) # Also, copy our pre-processed annotations to the dataset folder. # Note: you can find script for generating such annotations in attachments copyfile('nus_wide/small_test.json', os.path.join(img_folder, 'small_test.json')) copyfile('nus_wide/small_train.json', os.path.join(img_folder, 'small_train.json')) # # 数据集加载 # Dataloader，标签二值化. class NusDataset(Dataset): def __init__(self, data_path, anno_path, transforms): self.transforms = transforms with open(anno_path) as fp: json_data = json.load(fp) samples = json_data['samples'] self.classes = json_data['labels'] self.imgs = [] self.annos = [] self.data_path = data_path print('loading', anno_path) for sample in samples: self.imgs.append(sample['image_name']) self.annos.append(sample['image_labels']) for item_id in range(len(self.annos)): item = self.annos[item_id] vector = [cls in item for cls in self.classes] self.annos[item_id] = np.array(vector, dtype=float) def __getitem__(self, item): anno = self.annos[item] img_path = os.path.join(self.data_path, self.imgs[item]) img = Image.open(img_path) if self.transforms is not None: img = self.transforms(img) return img, anno def __len__(self): return len(self.imgs) # 数据集可视化 dataset_val = NusDataset(img_folder, os.path.join(img_folder, 'small_test.json'), None) dataset_train = NusDataset(img_folder, os.path.join(img_folder, 'small_train.json'), None) # def show_sample(img, binary_img_labels): # Convert the binary labels back to the text representation. img_labels = np.array(dataset_val.classes)[np.argwhere(binary_img_labels > 0)[:, 0]] plt.imshow(img) plt.title("{}".format(', '.join(img_labels))) plt.axis('off') plt.show() for sample_id in range(5): show_sample(*dataset_val[sample_id]) # 统计数据集的标签分布 samples = dataset_val.annos + dataset_train.annos samples = np.array(samples) with printoptions(precision=3, suppress=True): class_counts = np.sum(samples, axis=0) # Sort labels according to their frequency in the dataset. sorted_ids = np.array([i[0] for i in sorted(enumerate(class_counts), key=lambda x: x[1])], dtype=int) print('Label distribution (count, class name):', list(zip(class_counts[sorted_ids].astype(int), np.array(dataset_val.classes)[sorted_ids]))) plt.barh(range(len(dataset_val.classes)), width=class_counts[sorted_ids]) plt.yticks(range(len(dataset_val.classes)), np.array(dataset_val.classes)[sorted_ids]) plt.gca().margins(y=0) plt.grid() plt.title('Label distribution') plt.show()3. 创建模型3.1. 模型结构模型的 backbone 网络结构，采用 torchvision 中的 ResNeXt50 结构. 并修改其输出层，以适应 multi-label 分类任务.不同的是，ResNeXt50 的标准输出为 1000 个类(ImageNet)，而这里是 27 个. 此外，替换 softmax 函数为 sigmoid 函数. 后面会介绍原因.class Resnext50(nn.Module): def __init__(self, n_classes): super().__init__() resnet = models.resnext50_32x4d(pretrained=True) resnet.fc = nn.Sequential( nn.Dropout(p=0.2), nn.Linear(in_features=resnet.fc.in_features, out_features=n_classes) ) self.base_model = resnet self.sigm = nn.Sigmoid() def forward(self, x): return self.sigm(self.base_model(x))3.2. 损失函数在数据集确定及定义模型结构后，剩下的事情是选择损失函数. 由于是分类任务，其选择看似比较明显 - 交叉熵损失函数(CrossEntropy Loss). 但，这里会说明为什么其不适用于 multi-label 问题.图 - 模型采用 Softmax 分类器及交叉熵损失函数.上图给出了在各标签互斥的 multi-class 分类场景中交叉熵损失函数的计算. 在损失函数计算时，仅关注和 GT 标签对应的 logit，以及其与其它标签的差异性. 例如，上图中，损失函数值为： $-log(0.08)=2.52$Softmax 函数会将所有预测结果的概率和置为 1，因此，其不能处理存在多个正确标签的情况.图 - 模型采用 Sigmoid 分类器显而易见的方案是，将每个预测分别独立对待. 例如，采用 Sigmoid 函数对每个 logit 值分别归一化.Multi-label 问题中，会存在多个正确标签以及每个标签对应的预测概率. 然后，即可采用 BinaryCrossEntropy 损失函数计算每个标签的概率和 GT 之间的偏差.图 - 模型采用 Sigmoid 分类器和 BinaryCrossEntropyLoss.因此，最终采用 BinaryCrossEntropy 损失函数.criterion = nn.BCELoss()Sigmoid 和 Softmax 的计算对比:至此，剩下的唯一问题就是，在预测截断如何处理预测的概率. 通用做法是设置阈值，如果输出标签的概率值大于阈值，则表示为预测标签，否则跳过. 这里设置阈值为 0.5.3.3. 度量方式Metrics采用 sklearn.metrics.precision_score 计算度量，其参数设定为 average='macro'、average='micro' 或 average='samples'.def calculate_metrics(pred, target, threshold=0.5): pred = np.array(pred > threshold, dtype=float) return {'micro/precision': precision_score(y_true=target, y_pred=pred, average='micro'), 'micro/recall': recall_score(y_true=target, y_pred=pred, average='micro'), 'micro/f1': f1_score(y_true=target, y_pred=pred, average='micro'), 'macro/precision': precision_score(y_true=target, y_pred=pred, average='macro'), 'macro/recall': recall_score(y_true=target, y_pred=pred, average='macro'), 'macro/f1': f1_score(y_true=target, y_pred=pred, average='macro'), 'samples/precision': precision_score(y_true=target, y_pred=pred, average='samples'), 'samples/recall': recall_score(y_true=target, y_pred=pred, average='samples'), 'samples/f1': f1_score(y_true=target, y_pred=pred, average='samples'), }3.4. 模型训练实现如下：# 初始化训练参数 num_workers = 8 # Number of CPU processes for data preprocessing lr = 1e-4 # Learning rate batch_size = 32 save_freq = 1 # Save checkpoint frequency (epochs) test_freq = 200 # Test model frequency (iterations) max_epoch_number = 35 # Number of epochs for training # Note: on the small subset of data overfitting happens after 30-35 epochs mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] device = torch.device('cuda') # Save path for checkpoints save_path = 'chekpoints/' # Save path for logs logdir = 'logs/' # 辅助函数，断点保存 def checkpoint_save(model, save_path, epoch): f = os.path.join(save_path, 'checkpoint-{:06d}.pth'.format(epoch)) if 'module' in dir(model): torch.save(model.module.state_dict(), f) else: torch.save(model.state_dict(), f) print('saved checkpoint:', f) #数据处理 # Test preprocessing val_transform = transforms.Compose([ transforms.Resize((256, 256)), transforms.ToTensor(), transforms.Normalize(mean, std) ]) print(tuple(np.array(np.array(mean)*255).tolist())) # Train preprocessing train_transform = transforms.Compose([ transforms.Resize((256, 256)), transforms.RandomHorizontalFlip(), transforms.ColorJitter(), transforms.RandomAffine( degrees=20, translate=(0.2, 0.2), scale=(0.5, 1.5), shear=None, resample=False, fillcolor=tuple(np.array(np.array(mean)*255).astype(int).tolist())), transforms.ToTensor(), transforms.Normalize(mean, std) ]) # Initialize the dataloaders for training. test_annotations = os.path.join(img_folder, 'small_test.json') train_annotations = os.path.join(img_folder, 'small_train.json') test_dataset = NusDataset(img_folder, test_annotations, val_transform) train_dataset = NusDataset(img_folder, train_annotations, train_transform) train_dataloader = DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers, shuffle=True, drop_last=True) test_dataloader = DataLoader(test_dataset, batch_size=batch_size, num_workers=num_workers) num_train_batches = int(np.ceil(len(train_dataset) / batch_size)) # Initialize the model model = Resnext50(len(train_dataset.classes)) # Switch model to the training mode and move it to GPU. model.train() model = model.to(device) optimizer = torch.optim.Adam(model.parameters(), lr=lr) # If more than one GPU is available we can use both to speed up the training. if torch.cuda.device_count() > 1: model = nn.DataParallel(model) os.makedirs(save_path, exist_ok=True) # Loss function criterion = nn.BCELoss() # Tensoboard logger logger = SummaryWriter(logdir) # Run training epoch = 0 iteration = 0 while True: batch_losses = [] for imgs, targets in train_dataloader: imgs, targets = imgs.to(device), targets.to(device) optimizer.zero_grad() model_result = model(imgs) loss = criterion(model_result, targets.type(torch.float)) batch_loss_value = loss.item() loss.backward() optimizer.step() logger.add_scalar('train_loss', batch_loss_value, iteration) batch_losses.append(batch_loss_value) with torch.no_grad(): result = calculate_metrics(model_result.cpu().numpy(), targets.cpu().numpy()) for metric in result: logger.add_scalar('train/' + metric, result[metric], iteration) if iteration % test_freq == 0: model.eval() with torch.no_grad(): model_result = [] targets = [] for imgs, batch_targets in test_dataloader: imgs = imgs.to(device) model_batch_result = model(imgs) model_result.extend(model_batch_result.cpu().numpy()) targets.extend(batch_targets.cpu().numpy()) result = calculate_metrics(np.array(model_result), np.array(targets)) for metric in result: logger.add_scalar('test/' + metric, result[metric], iteration) print("epoch:{:2d} iter:{:3d} test: " "micro f1: {:.3f} " "macro f1: {:.3f} " "samples f1: {:.3f}".format(epoch, iteration, result['micro/f1'], result['macro/f1'], result['samples/f1'])) model.train() iteration += 1 # loss_value = np.mean(batch_losses) print("epoch:{:2d} iter:{:3d} train: loss:{:.3f}".format(epoch, iteration, loss_value)) if epoch % save_freq == 0: checkpoint_save(model, save_path, epoch) epoch += 1 if max_epoch_number < epoch: break在 1080Ti 显卡上训练约 1 个小时，35 个 epochs直到过拟合. 最高 macro F1-score 为 0.520，其对应的最佳 micro F1-score 为 0.666. 存在的差异性的原因可能是数据相当不均衡所导致.3.5. 模型预测# Run inference on the test data model.eval() for sample_id in [1,2,3,4,6]: test_img, test_labels = test_dataset[sample_id] test_img_path = os.path.join(img_folder, test_dataset.imgs[sample_id]) with torch.no_grad(): raw_pred = model(test_img.unsqueeze(0)).cpu().numpy()[0] raw_pred = np.array(raw_pred > 0.5, dtype=float) predicted_labels = np.array(dataset_val.classes)[np.argwhere(raw_pred > 0)[:, 0]] if not len(predicted_labels): predicted_labels = ['no predictions'] img_labels = np.array(dataset_val.classes)[np.argwhere(test_labels > 0)[:, 0]] plt.imshow(Image.open(test_img_path)) plt.title("Predicted labels: {} \nGT labels: {}".format(', '.join(predicted_labels), ', '.join(img_labels))) plt.axis('off') plt.show()如图：4. 总结这里主要简单介绍了关于 multi-label 分类问题的解决方案.后面可以尝试更多 SoTA 论文，以及不同的损失函数、额外的处理等，以提升精度. 例如，使用当前热门的 GCN 为网络提供标签间的组合关系等信息，可能更合适.5. 相关资料[1] - Multi-Label Text Classification[2] - Deep dive into multi-label classification[3] - Multi-label vs. Multi-class Classification: Sigmoid vs. Softmax
图像标注-基于PyTorch的多输出图像分类[译]
原文：Multi-Label Image Classification with PyTorch - 2020.04.04作者：Dmitry Retinskiy (Xperience.AI)实现：Github - PyTorch-Multi-Label-Image-Classification自从 2012 年 ImageNet 竞赛中神经网络第一次赢得比赛， Alex Krizhevsky, Ilya Sutskever 和 Geoffrey Hinton 彻底改变了图像分类领域.如今，图像的单标签任务(即，图像识别)，已经被广泛研究和解决. 然而，现实场景中，往往不局限于“每张图片仅有一个标签” 的任务，某些时候，每张图片会有更多标签.对此，这里研究的是多输出图像分类(multi-output classification) 或图像标注(image tagging).1. 多标签分类的场景在图像分类领域中，可能会遇到这样的场景：判断目标的集中属性. 例如，类别、颜色、尺寸、等等. 相比较于传统图像分类，其输出将会包含2个及以上的属性.这里关注的问题是，提前已知属性的数量(the number of the properties). 这种任务被称为多输出分类. 实际上，其是多标签分类(multi-label classification)的一种特殊形式，后者处理的也是预测几种属性，但其数量对于每个样本可能都是不同的.2. 数据集Kaggle - Fashion Product Images. 其共有超过 44000 张服装图像，每个图像包含有 9 个标签. 这里仅采用三个标签：gender, articleType 和 baseColour. 示例如下：数据提取后，分布情况为：[1] - 5 个 gender 标签：Boys, Girls, Men, Unisex, Women[2] - 47 种 colors[3] - 143 种类目，如Sports Sandals, Wallets, Sweaters.基于该数据集，下面会创建神经网络模型，并进行训练，预测数据集中图片的三种标签(gender, article, and color).2.1. 数据集划分数据集共采用 40000 张图像，其中 320000 训练数据集，8000 测试数据集. split_data.py:import argparse import csv import os import numpy as np from PIL import Image from tqdm import tqdm def save_csv(data, path, fieldnames=['image_path', 'gender', 'articleType', 'baseColour']): with open(path, 'w', newline='') as csv_file: writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer.writeheader() for row in data: writer.writerow(dict(zip(fieldnames, row))) def split_data(): input_folder = "fashion-product-images" output_folder = "fashion-product-images" annotation = os.path.join(input_folder, 'styles.csv') # open annotation file all_data = [] with open(annotation) as csv_file: reader = csv.DictReader(csv_file) # each row in the CSV file corresponds to the image for row in tqdm(reader, total=reader.line_num): # we need image ID to build the path to the image file img_id = row['id'] # gender = row['gender'] articleType = row['articleType'] baseColour = row['baseColour'] img_name = os.path.join(input_folder, 'images', str(img_id) + '.jpg') if os.path.exists(img_name): # check if the image has 80*60 pixels with 3 channels img = Image.open(img_name) if img.size == (60, 80) and img.mode == "RGB": all_data.append([img_name, gender, articleType, baseColour]) # for reproduce the results np.random.seed(42) # construct a Numpy array from the list all_data = np.asarray(all_data) # Take 40000 samples in random order inds = np.random.choice(40000, 40000, replace=False) # split the data into train/val and save them as csv files save_csv(all_data[inds][:32000], os.path.join(output_folder, 'train.csv')) save_csv(all_data[inds][32000:40000], os.path.join(output_folder, 'val.csv')) if __name__ == '__main__': split_data()3. 构建模型3.1. 数据加载由于每张图片不止一个标注标签，因此需要调整数据读取和加载到内存的方式. 创建继承自 PyTorch Dataset 的函数类，以便于解析标注数据，并提取标签. multi-output 分类和 single-class 分类的关键区别是：返回数据集中每个样本的多个标签.dataset.py:import csv import numpy as np from PIL import Image from torch.utils.data import Dataset mean = [0.485, 0.456, 0.406] std = [0.229, 0.224, 0.225] class AttributesDataset(): def __init__(self, annotation_path): color_labels = [] gender_labels = [] article_labels = [] with open(annotation_path) as f: reader = csv.DictReader(f) for row in reader: color_labels.append(row['baseColour']) gender_labels.append(row['gender']) article_labels.append(row['articleType']) self.color_labels = np.unique(color_labels) self.gender_labels = np.unique(gender_labels) self.article_labels = np.unique(article_labels) self.num_colors = len(self.color_labels) self.num_genders = len(self.gender_labels) self.num_articles = len(self.article_labels) self.color_id_to_name = dict(zip(range(len(self.color_labels)), self.color_labels)) self.color_name_to_id = dict(zip(self.color_labels, range(len(self.color_labels)))) self.gender_id_to_name = dict(zip(range(len(self.gender_labels)), self.gender_labels)) self.gender_name_to_id = dict(zip(self.gender_labels, range(len(self.gender_labels)))) self.article_id_to_name = dict(zip(range(len(self.article_labels)), self.article_labels)) self.article_name_to_id = dict(zip(self.article_labels, range(len(self.article_labels)))) # class FashionDataset(Dataset): def __init__(self, annotation_path, attributes, transform=None): super().__init__() self.transform = transform self.attr = attributes # initialize the arrays to store the ground truth labels and paths to the images self.data = [] self.color_labels = [] self.gender_labels = [] self.article_labels = [] # read the annotations from the CSV file with open(annotation_path) as f: reader = csv.DictReader(f) for row in reader: self.data.append(row['image_path']) self.color_labels.append(self.attr.color_name_to_id[row['baseColour']]) self.gender_labels.append(self.attr.gender_name_to_id[row['gender']]) self.article_labels.append(self.attr.article_name_to_id[row['articleType']]) def __len__(self): return len(self.data) def __getitem__(self, idx): # take the data sample by its index img_path = self.data[idx] img = Image.open(img_path) # if self.transform: img = self.transform(img) # return the image and all the associated labels dict_data = { 'img': img, 'labels': { 'color_labels': self.color_labels[idx], 'gender_labels': self.gender_labels[idx], 'article_labels': self.article_labels[idx] } } return dict_data 3.2. 数据增强数据增强和随机变换，有助于避免网络训练时的过拟合问题.训练阶段主要采用：random flipping, slight color modifications, rotation, scaling, 和 translation (unified in an affine transformation). 此外，在加载进网络前，进行数据归一化.train_transform = transforms.Compose([ transforms.RandomHorizontalFlip(p=0.5), transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0), transforms.RandomAffine(degrees=20, translate=(0.1, 0.1), scale=(0.8, 1.2), shear=None, resample=False, fillcolor=(255, 255, 255)), transforms.ToTensor(), transforms.Normalize(mean, std) ])验证阶段，不对数据进行随机化处理，仅归一化及格式转换操作：val_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean, std) ])3.3. 模型构建首先是模型类定义. 以 torchvision.models 中的 mobilenet_v2 为例. mobilenet_v2 是单标签分类器.为了适应 multi-output 任务，调整 mobilenet_v2 网络，以同时预测三种属性，其 head 网络分别为：color，gender 和 article. 每个 head 分别接对应的 cross-entropy loss. 如图：具体实现为 - model.py：import torch import torch.nn as nn import torch.nn.functional as F import torchvision.models as models class MultiOutputModel(nn.Module): def __init__(self, n_color_classes, n_gender_classes, n_article_classes): super().__init__() # take the model without classifier self.base_model = models.mobilenet_v2().features # size of the layer before classifier last_channel = models.mobilenet_v2().last_channel # the input for the classifier should be two-dimensional, but we will have # [batch_size, channels, width, height] # so, let's do the spatial averaging: reduce width and height to 1 self.pool = nn.AdaptiveAvgPool2d((1, 1)) # create separate classifiers for our outputs self.color = nn.Sequential( nn.Dropout(p=0.2), nn.Linear(in_features=last_channel, out_features=n_color_classes) ) self.gender = nn.Sequential( nn.Dropout(p=0.2), nn.Linear(in_features=last_channel, out_features=n_gender_classes) ) self.article = nn.Sequential( nn.Dropout(p=0.2), nn.Linear(in_features=last_channel, out_features=n_article_classes) ) def forward(self, x): x = self.base_model(x) x = self.pool(x) # reshape from [batch, channels, 1, 1] to [batch, channels] to put it into classifier x = torch.flatten(x, 1) return { 'color': self.color(x), 'gender': self.gender(x), 'article': self.article(x) } def get_loss(self, net_output, ground_truth): color_loss = F.cross_entropy(net_output['color'], ground_truth['color_labels']) gender_loss = F.cross_entropy(net_output['gender'], ground_truth['gender_labels']) article_loss = F.cross_entropy(net_output['article'], ground_truth['article_labels']) loss = color_loss + gender_loss + article_loss return loss, {'color': color_loss, 'gender': gender_loss, 'article': article_loss}3.4. 模型训练multi-output 分类网络的训练与 single-output 分类任务是相同的，具体可参考：Image Classification using Transfer Learning in PyTorch首先定义几个辅助函数：import matplotlib.pyplot as plt # def visualize_gt_data(dataset, attributes): imgs = [] gt_labels = [] n_cols = 5 n_rows = 3 # store the original transforms from the dataset transforms = dataset.transform # and not use them during visualization dataset.transform = None for img_idx in range(n_cols * n_rows): sample = dataset[img_idx] img = sample['img'] labels = sample['labels'] gt_color = attributes.color_id_to_name[labels['color_labels']] gt_gender = attributes.gender_id_to_name[labels['gender_labels']] gt_article = attributes.article_id_to_name[labels['article_labels']] imgs.append(img) gt_labels.append("{}\n{}\n{}".format(gt_gender, gt_article, gt_color)) title = "Ground truth labels" fig, axs = plt.subplots(n_rows, n_cols, figsize=(10, 10)) axs = axs.flatten() for img, ax, label in zip(imgs, axs, gt_labels): ax.set_xlabel(label, rotation=0) ax.get_xaxis().set_ticks([]) ax.get_yaxis().set_ticks([]) ax.imshow(img) plt.suptitle(title) plt.tight_layout() plt.show() # restore original transforms dataset.transform = transforms具体实现如 - train.py：import argparse import os from datetime import datetime import torch import torchvision.transforms as transforms from torch.utils.data import DataLoader from torch.utils.tensorboard import SummaryWriter def get_cur_time(): return datetime.strftime(datetime.now(), '%Y-%m-%d_%H-%M') def checkpoint_save(model, name, epoch): f = os.path.join(name, 'checkpoint-{:06d}.pth'.format(epoch)) torch.save(model.state_dict(), f) print('Saved checkpoint:', f) return f def train(start_epoch=1, N_epochs=50, batch_size=16, num_workers=8): attributes_file = 'fashion-product-images/styles.csv' device = torch.device('cuda') # attributes variable contains labels for the categories in the dataset and mapping between string names and IDs attributes = AttributesDataset(attributes_file) # specify image transforms for augmentation during training train_transform = transforms.Compose([ transforms.RandomHorizontalFlip(p=0.5), transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0), transforms.RandomAffine(degrees=20, translate=(0.1, 0.1), scale=(0.8, 1.2), shear=None, resample=False, fillcolor=(255, 255, 255)), transforms.ToTensor(), transforms.Normalize(mean, std) ]) # during validation we use only tensor and normalization transforms val_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean, std) ]) train_dataset = FashionDataset('fashion-product-images/train.csv', attributes, train_transform) train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) val_dataset = FashionDataset('fashion-product-images/val.csv', attributes, val_transform) val_dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers) model = MultiOutputModel(n_color_classes=attributes.num_colors, n_gender_classes=attributes.num_genders, n_article_classes=attributes.num_articles)\ .to(device) optimizer = torch.optim.Adam(model.parameters()) logdir = os.path.join('logs', get_cur_time()) print(logdir) savedir = os.path.join('checkpoints', get_cur_time()) print(savedir) os.makedirs(logdir, exist_ok=False) os.makedirs(savedir, exist_ok=False) logger = SummaryWriter(logdir) n_train_samples = len(train_dataloader) visualize_gt_data(val_dataset, attributes) print("\nAll gender labels:\n", attributes.gender_labels) print("\nAll color labels:\n", attributes.color_labels) print("\nAll article labels:\n", attributes.article_labels) print("Starting training ...") checkpoint_path = None for epoch in range(start_epoch, N_epochs + 1): total_loss = 0 accuracy_color = 0 accuracy_gender = 0 accuracy_article = 0 for batch in train_dataloader: optimizer.zero_grad() img = batch['img'] target_labels = batch['labels'] target_labels = {t: target_labels[t].to(device) for t in target_labels} output = model(img.to(device)) loss_train, losses_train = model.get_loss(output, target_labels) total_loss += loss_train.item() batch_accuracy_color, batch_accuracy_gender, batch_accuracy_article = \ calculate_metrics(output, target_labels) accuracy_color += batch_accuracy_color accuracy_gender += batch_accuracy_gender accuracy_article += batch_accuracy_article loss_train.backward() optimizer.step() print("epoch {:4d}, loss: {:.4f}, color: {:.4f}, gender: {:.4f}, article: {:.4f}".format( epoch, total_loss / n_train_samples, accuracy_color / n_train_samples, accuracy_gender / n_train_samples, accuracy_article / n_train_samples)) logger.add_scalar('train_loss', total_loss / n_train_samples, epoch) if epoch % 25 == 0: checkpoint_path = checkpoint_save(model, savedir, epoch) if epoch % 5 == 0: validate(model, val_dataloader, device, logger, epoch) checkpoint_path = checkpoint_save(model, savedir, epoch - 1) return checkpoint_path3.5. 模型验证首先，回顾下 single-output 分类任务中度量标准 - 准确度(accuracy)，其计算比较简单，所有数据中，模型所正确预测的结果.$$ accuracy = \frac{correct \ predictions}{dataset \ size} $$那么，对于 multi-output 分类任务呢，仍然是可以使用准确度作为度量标准的. 只需每个输出单独计算.import warnings from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, accuracy_score def calculate_metrics(output, target): _, predicted_color = output['color'].cpu().max(1) gt_color = target['color_labels'].cpu() _, predicted_gender = output['gender'].cpu().max(1) gt_gender = target['gender_labels'].cpu() _, predicted_article = output['article'].cpu().max(1) gt_article = target['article_labels'].cpu() with warnings.catch_warnings(): # sklearn may produce a warning when processing zero row in confusion matrix warnings.simplefilter("ignore") accuracy_color = accuracy_score(y_true=gt_color.numpy(), y_pred=predicted_color.numpy()) accuracy_gender = accuracy_score(y_true=gt_gender.numpy(), y_pred=predicted_gender.numpy()) accuracy_article = accuracy_score(y_true=gt_article.numpy(), y_pred=predicted_article.numpy()) return accuracy_color, accuracy_gender, accuracy_article ## def checkpoint_load(model, name): print('Restoring checkpoint: {}'.format(name)) model.load_state_dict(torch.load(name, map_location='cpu')) epoch = int(os.path.splitext(os.path.basename(name))[0].split('-')[1]) return epoch def net_output_to_predictions(output): _, predicted_colors = output['color'].cpu().max(1) _, predicted_genders = output['gender'].cpu().max(1) _, predicted_articles = output['article'].cpu().max(1) return predicted_colors.numpy().tolist(), predicted_genders.numpy().tolist(), predicted_articles.numpy().tolist() def validate(model, dataloader, device, logger=None, epoch=None, checkpoint=None): if checkpoint is not None: checkpoint_load(model, checkpoint) model.eval() color_predictions = [] gender_predictions = [] article_predictions = [] with torch.no_grad(): avg_loss = 0 accuracy_color = 0 accuracy_gender = 0 accuracy_article = 0 for batch in dataloader: img = batch['img'] target_labels = batch['labels'] target_labels = {t: target_labels[t].to(device) for t in target_labels} output = model(img.to(device)) val_train, val_train_losses = model.get_loss(output, target_labels) avg_loss += val_train.item() batch_accuracy_color, batch_accuracy_gender, batch_accuracy_article = \ calculate_metrics(output, target_labels) accuracy_color += batch_accuracy_color accuracy_gender += batch_accuracy_gender accuracy_article += batch_accuracy_article (batch_color_predictions, batch_gender_predictions, batch_article_predictions) = net_output_to_predictions(output) color_predictions.extend(batch_color_predictions) gender_predictions.extend(batch_gender_predictions) article_predictions.extend(batch_article_predictions) n_samples = len(dataloader) avg_loss /= n_samples accuracy_color /= n_samples accuracy_gender /= n_samples accuracy_article /= n_samples print('-' * 72) print("Validation loss: {:.4f}, color: {:.4f}, gender: {:.4f}, article: {:.4f}\n".format( avg_loss, accuracy_color, accuracy_gender, accuracy_article)) if logger is not None and epoch is not None: logger.add_scalar("val_loss", avg_loss, epoch) logger.add_scalar("val_accuracy/color", accuracy_color, epoch) logger.add_scalar("val_accuracy/gender", accuracy_gender, epoch) logger.add_scalar("val_accuracy/article", accuracy_article, epoch) model.train() return color_predictions, gender_predictions, article_predictions3.6. 模型评价混淆矩阵.test.py:import argparse import os import numpy as np import torch import torchvision.transforms as transforms from torch.utils.data import DataLoader def visualize_grid(dataset, attributes, color_predictions, gender_predictions, article_predictions): imgs = [] labels = [] predicted_color_all = [] predicted_gender_all = [] predicted_article_all = [] gt_labels = [] gt_color_all = [] gt_gender_all = [] gt_article_all = [] # store the original transforms from the dataset transforms = dataset.transform # and not use them during visualization dataset.transform = None for (sample, predicted_color, predicted_gender, predicted_article) in zip( dataset, color_predictions, gender_predictions, article_predictions): predicted_color = attributes.color_id_to_name[predicted_color] predicted_gender = attributes.gender_id_to_name[predicted_gender] predicted_article = attributes.article_id_to_name[predicted_article] gt_color = attributes.color_id_to_name[sample['labels']['color_labels']] gt_gender = attributes.gender_id_to_name[sample['labels']['gender_labels']] gt_article = attributes.article_id_to_name[sample['labels']['article_labels']] predicted_color_all.append(predicted_color) predicted_gender_all.append(predicted_gender) predicted_article_all.append(predicted_article) gt_color_all.append(gt_color) gt_gender_all.append(gt_gender) gt_article_all.append(gt_article) imgs.append(sample['img']) labels.append("{}\n{}\n{}".format(predicted_gender, predicted_article, predicted_color)) gt_labels.append("{}\n{}\n{}".format(gt_gender, gt_article, gt_color)) # restore original transforms dataset.transform = transforms # Draw confusion matrices # color cn_matrix = confusion_matrix( y_true=gt_color_all, y_pred=predicted_color_all, labels=attributes.color_labels, normalize='true') plt.rcParams.update({'font.size': 5}) plt.rcParams.update({'figure.dpi': 300}) ConfusionMatrixDisplay(cn_matrix, attributes.color_labels).plot( include_values=False, xticks_rotation='vertical') plt.title("Colors") plt.tight_layout() plt.show() # gender cn_matrix = confusion_matrix( y_true=gt_gender_all, y_pred=predicted_gender_all, labels=attributes.gender_labels, normalize='true') ConfusionMatrixDisplay(cn_matrix, attributes.gender_labels).plot( xticks_rotation='horizontal') plt.title("Genders") plt.tight_layout() plt.show() plt.rcParams.update({'font.size': 2.5}) cn_matrix = confusion_matrix( y_true=gt_article_all, y_pred=predicted_article_all, labels=attributes.article_labels, normalize='true') ConfusionMatrixDisplay(cn_matrix, attributes.article_labels).plot( include_values=False, xticks_rotation='vertical') plt.title("Article types") plt.show() plt.rcParams.update({'font.size': 5}) plt.rcParams.update({'figure.dpi': 100}) title = "Predicted labels" n_cols = 5 n_rows = 3 fig, axs = plt.subplots(n_rows, n_cols, figsize=(10, 10)) axs = axs.flatten() for img, ax, label in zip(imgs, axs, labels): ax.set_xlabel(label, rotation=0) ax.get_xaxis().set_ticks([]) ax.get_yaxis().set_ticks([]) ax.imshow(img) plt.suptitle(title) plt.tight_layout() plt.show() # def test(checkpoint_path): attributes_file = 'fashion-product-images/styles.csv' device = torch.device("cuda") # attributes variable contains labels for the categories in the dataset and mapping between string names and IDs attributes = AttributesDataset(attributes_file) # during validation we use only tensor and normalization transforms val_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean, std) ]) test_dataset = FashionDataset('fashion-product-images/val.csv', attributes, val_transform) test_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=False, num_workers=8) model = MultiOutputModel(n_color_classes=attributes.num_colors, n_gender_classes=attributes.num_genders, n_article_classes=attributes.num_articles).to(device) model_predictions = validate(model, test_dataloader, device, checkpoint=checkpoint_path) # Visualization of the trained model visualize_grid(test_dataset, attributes, *model_predictions)\ if __name__ == '__main__': test(last_checkpoint_path)测试结果示例：color混淆矩阵如下：可以看出模型对于比较相似的颜色产生了混淆，例如：magenta, pink 和 purple，即使对于人而言，也很难识别数据集中的47种颜色. 比如，因此，对于颜色来说，低准确度不是大问题的. 如果项解决该问题，只需减少数据集中的颜色种类数，例如，10 种，重新将相似颜色合并为一个类，然后再次训练模型. 应该就可以得到更好的结果.gender 混淆矩阵如下：可以看出，模型混淆了 ‘girls’ 和 ‘women’ 、“men” 和 “unisex” 标签. 同样的，对于人而言，这些都可能很难区分.clothes 和 accessories 的混淆矩阵如下：某些类目确实是难以区分，比如下面的两个 bags:4. 总结这里主要是对基于 single-output 模型进行 multi-output 模型的构建进行介绍. 此外，还介绍了基于混淆矩阵的模型预测结果的验证.
End-to-End Multi-label Classification多标签分类[译]
原文：End-to-End Multi-label Classification - 2020.05.08Kaggle：Example of multi-label multi-class classification作者：Bhartendu1. Multi-Label classification在机器学习领域，多标签分类及其强相关的多输出分类(multi-output classification)是分类问题的一种变形，其针对的是每个实例可以被指定多个类别标签. 多标签分类是多类别分类(multiclass classification) 的一种泛化，后者是将实例精确地分类到多个类别(超过两个)中的某个标签的单标签问题. 而，对于多标签分类问题，没有限定实例能被指定的类别标签数量.正式地，多标签分类针对的问题是，构建输入 $\mathbf{x}$ 到二值向量 $\mathbf{y}$ 的模型，其中，向量 $\mathbf{y}$ 中的每个元素(标签)的值为 0 或 1.这里举例说明多标签问题:服装主体包含多个属性(标签)，如颜色、图案、类目、场景等. 每个属性有其自己的类别，比如，场景可以被分类为运动(Sports)、正式(Formal)、民族(Ethnic)、休闲(Casual). 可以采用的数据集有：DeepFashion, FashionGen. 这里以 Kaggle fashion_small 数据集为例.2. Fashion_small 数据探索fashion_small 数据集共包含 21K 服装主体. 这里对每个标签中的每个类设置最小样本数 $\mathbf{s}$. 对于每个标签中的所有类设置固定的数量.最小样本数：$\mathcal{s} = 700$.如图，为每个标签的条形图(移除数量小于 $\mathcal{s}$ 的类)：Attributes | Number of classes ------------------------------------------------- gender | 3 subCategory | 2 articleType | 5 baseColour | 8 season | 2 usage | 4 -------------------------------------------------- Total number of label | 6 Total number of classes | 24 Total number of images | 10809样例图片如下(不同标签的不同类)：最终的数据组织形式为(DataFrame)：因此，最终服装主体识别问题为，多标签(6个)多类别(24个)问题.2.1. 数据探索完整实现该部分的代码实现为：import os import json import warnings warnings.filterwarnings('ignore') import pandas as pd import numpy as np import seaborn as sns import matplotlib.image as mpimg import matplotlib.pyplot as plt from collections import Counter, defaultdict from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder from keras.preprocessing import image from keras.models import Model, load_model from keras.layers import Input, Dense, Dropout, Conv2D, MaxPooling2D from keras.layers import BatchNormalization, add, GlobalAveragePooling2D from keras.callbacks import EarlyStopping, ModelCheckpoint from keras.utils.np_utils import to_categorical from keras.utils import plot_model # path = '/path/to/fashion_small/' table = 'styles.csv' base = path+'resized_images/' print('\nDirectory contains:\n', os.listdir(path)) data = pd.read_csv(path+table, error_bad_lines=False, warn_bad_lines=False) print('\n CSV contains {} entries, with attributes\n {}'.format(len(data),data.keys().values)) ''' CSV contains 44424 entries, with attributes ['id' 'gender' 'masterCategory' 'subCategory' 'articleType' 'baseColour' 'season' 'year' 'usage' 'productDisplayName'] ''' # Check for exsisting images in database im_path = [] for item_id in data['id']: tmp_path = base + str(item_id) +'.jpg' if os.path.exists(tmp_path): im_path.append(tmp_path) else: data.drop(data[data.id == item_id].index, inplace=True) print('Item {} Doesn\'t exists'.format(item_id)) data['img_path'] = im_path data = data.sample(frac=1, random_state=10) data.dropna(inplace=True) data.reset_index(drop=True, inplace=True) data.tail() ''' Item 39403 Doesn't exists Item 39410 Doesn't exists Item 39401 Doesn't exists Item 39425 Doesn't exists Item 12347 Doesn't exists ''' #Unique values per column data.nunique() ''' id 44072 gender 5 masterCategory 7 subCategory 45 articleType 141 baseColour 46 season 4 year 13 usage 8 productDisplayName 30801 img_path 44072 dtype: int64 ''' # Analysing Apparels from masterCategory # a. Removing Duplicate entries # b. Removing 'year' :: Can't be learnt visually # c. Removing 'productDisplayName' :: Can't be learnt visually df = data[data['masterCategory'] == 'Apparel'] df.drop_duplicates(subset='img_path', inplace=True) df.drop(columns=['masterCategory', 'year', 'productDisplayName'], inplace=True) df.reset_index(inplace=True, drop=True) df.tail(5)辅助函数：# Temp functions # a. Display Categorical charts # b. Clean dataFrame as per minimum requirements for modelling def disp_category(category, df, min_items=500): ''' Display Categorical charts category :: dtpye=str, Category (attribute/column name) df :: DataFrame to be loaded min_items :: dtype=int, minimum rows to qualify as class returns classes to be selected for the analysis ''' dff = df.groupby(category) class_ = dff.count().sort_values(by='id')['id'].reset_index() class_.columns=[category,'count'] class_= class_[class_['count']>=min_items][category] df = df[df[category].isin(class_)] labels = df[category] counts = defaultdict(int) for l in labels: counts[l] += 1 counts_df = pd.DataFrame.from_dict(counts, orient='index') counts_df.columns = ['count'] counts_df.sort_values('count', ascending=False, inplace=True) # fig, ax = plt.subplots() ax = sns.barplot(x=counts_df.index, y=counts_df['count'], ax=ax) fig.set_size_inches(10,5) ax.set_xticklabels(ax.xaxis.get_majorticklabels(), rotation=-90); ax.set_title('Label: '+category.upper()) plt.show() return class_ def reduce_df(df, category, class_): ''' Remove noise points from dataFrame category :: dtpye=str, Category (attribute/column name) df :: DataFrame to be loaded class_ :: list, classes to be seleted for the analysis returns: clean dataFrame ''' print('Analysing {} as feature'.format(category)) print('Prior Number of datapoints: {} :: Classes: {}'.format( len(df), set(df[category]))) df = df[df[category].isin(class_)] df.reset_index(drop=True, inplace=True) print('Posterior Number of datapoints: {} :: Classes: {}\n'.format( len(df), set(df[category]))) return df标签分析：# Analysing labels # Minmum requirement: set minimum samples per class for cate in df.keys()[1:-1]: class_ = disp_category(cate, df, min_items=700) df = reduce_df(df, cate, class_) ''' Analysing gender as feature Prior Number of datapoints: 21361 :: Classes: {'Boys', 'Girls', 'Unisex', 'Men', 'Women'} Posterior Number of datapoints: 20709 :: Classes: {'Boys', 'Men', 'Women'} Analysing subCategory as feature Prior Number of datapoints: 20709 :: Classes: {'Topwear', 'Loungewear and Nightwear', 'Bottomwear', 'Socks', 'Apparel Set', 'Saree', 'Innerwear', 'Dress'} Posterior Number of datapoints: 19307 :: Classes: {'Topwear', 'Bottomwear', 'Innerwear'} Analysing articleType as feature Prior Number of datapoints: 19307 :: Classes: {'Sweaters', 'Tights', 'Boxers', 'Churidar', 'Jeggings', 'Suspenders', 'Salwar', 'Swimwear', 'Rompers', 'Belts', 'Nehru Jackets', 'Shirts', 'Tracksuits', 'Skirts', 'Shapewear', 'Lehenga Choli', 'Waistcoat', 'Tunics', 'Sweatshirts', 'Track Pants', 'Briefs', 'Jeans', 'Salwar and Dupatta', 'Trunk', 'Bra', 'Rain Jacket', 'Dresses', 'Innerwear Vests', 'Trousers', 'Leggings', 'Blazers', 'Kurtis', 'Jackets', 'Camisoles', 'Capris', 'Tshirts', 'Shorts', 'Tops', 'Stockings', 'Dupatta', 'Kurtas', 'Shrug', 'Patiala'} Posterior Number of datapoints: 14322 :: Classes: {'Tops', 'Shirts', 'Kurtas', 'Tshirts', 'Briefs'} Analysing baseColour as feature Prior Number of datapoints: 14322 :: Classes: {'Maroon', 'Rose', 'Tan', 'Black', 'Peach', 'Beige', 'Pink', 'Fluorescent Green', 'Skin', 'Magenta', 'Charcoal', 'Blue', 'Lavender', 'Mushroom Brown', 'Turquoise Blue', 'Khaki', 'Sea Green', 'Silver', 'Navy Blue', 'Green', 'Brown', 'Mustard', 'Lime Green', 'Grey', 'Multi', 'Orange', 'Off White', 'White', 'Purple', 'Nude', 'Rust', 'Coffee Brown', 'Burgundy', 'Yellow', 'Gold', 'Cream', 'Mauve', 'Red', 'Olive', 'Grey Melange', 'Taupe', 'Teal'} Posterior Number of datapoints: 11141 :: Classes: {'Navy Blue', 'Blue', 'Grey', 'White', 'Red', 'Black', 'Purple', 'Green'} Analysing season as feature Prior Number of datapoints: 11141 :: Classes: {'Fall', 'Winter', 'Spring', 'Summer'} Posterior Number of datapoints: 10826 :: Classes: {'Fall', 'Summer'} Analysing usage as feature Prior Number of datapoints: 10826 :: Classes: {'Smart Casual', 'Casual', 'Party', 'Sports', 'Formal', 'Ethnic'} Posterior Number of datapoints: 10809 :: Classes: {'Sports', 'Formal', 'Ethnic', 'Casual'} ''' # Posterior Unique values per column df.nunique() ''' id 10809 gender 3 subCategory 2 articleType 5 baseColour 8 season 2 usage 4 img_path 10809 dtype: int64 '''采样示例图片：# Sample images (distinct labels with all classes) def grid_plot(title, grouped_df, samples=2): samples= len(grouped_df) item_id, img_path = grouped_df['id'].values, grouped_df['img_path'].values plt.figure(figsize=(7,15)) for i in range(samples): plt.subplot(len(item_id) / samples + 1, 3, i + 1, title=title) plt.axis('off') plt.imshow(mpimg.imread(img_path[i])) for cat in df.keys()[1:-1]: tmp_df = df.groupby(cat) for group_objects in tmp_df: title = '{}:\n {}'.format(cat,group_objects[0]) grouped_df = group_objects[1].sample(n=3) grid_plot(title, grouped_df) # Ready to use Dataframe df.tail()3. 建模(CNN Model)[1] - 构建 resudual block 模块，基于 ResNet 中的 Residual Block 如(Keras实现)：def custom_residual_block(self, r, channels): r = BatchNormalization()(r) r = Conv2D(channels, (3, 3), activation='relu')(r) h = Conv2D(channels, (3, 3), padding='same', activation='relu')(r) h = Conv2D(channels, (3, 3), padding='same', activation='relu')(h) h = Conv2D(channels, (3, 3), padding='same', activation='relu')(h) return add([h, r])[2] - 构建模型，从 Conv 层和 Pooling 层，再接 residual blocks，输出为 Dense 层.def build_model(self): input_layer = Input(shape=self.img_shape) # Conv Layers en_in = Conv2D(16, (5, 5), activation='relu')(input_layer) h = MaxPooling2D((3, 3))(en_in) h = self.custom_residual_block(h, 32) h = self.custom_residual_block(h, 32) h = self.custom_residual_block(h, 48) h = self.custom_residual_block(h, 48) h = self.custom_residual_block(h, 48) h = self.custom_residual_block(h, 48) h = self.custom_residual_block(h, 54) h = self.custom_residual_block(h, 54) en_out = self.custom_residual_block(h, 54) # Dense i-th label (to be defined for each label) label_i = self.custom_residual_block(en_out, 48) label_i = GlobalAveragePooling2D()(label_i) label_i = Dense(100, activation='relu')(label_i) label_i = Dropout(0.1)(label_i) label_i = Dense(50, activation='relu')(label_i) out_i = Dense(num_class_i, activation='softmax')(label_i) return Model(input_layer, [out_1 .... out_6])[3] - 组装并编译模型. 采用交叉熵作为损失函数；accuracy 作为性能度量.self.cnn_model = self.build_model() self.cnn_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])[4] - 模型可视化. 基于 keras.utils 的 plot_model 工具.plot_model(self.cnn_model, to_file='consolidated_model.png')[5] - 开始模型训练. 建议保留 callbacks 以保存最佳模型(或模型权重)，以及添加 EarlyStopping 以避免过拟合以及不必要的计算.def train_model(self, x_train, y_train, x_val, y_val, save_path, epochs=25, batch_size=16): # early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto', restore_best_weights=True) # check_pointer = ModelCheckpoint(save_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) # history = self.cnn_model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_val, y_val), callbacks=[early_stopping, check_pointer]) return history对所有标签都具有较好表现的模型，依赖于问题的视觉复杂度. 因为卷积层(convolutional filters)被要求同时(联合)的学习所有标签的 ROI，任何表现较弱的标签都会对模型的性能有不利影响(相比于独立的CNN 分类器).训练过程损失函数衰减和性能曲线如图，可以看出，某些属性的模型可以不用很深的网络即可获得相当不错的表现.可以看出，对于某些标签，模型学习的相当快(尤其是对于类别较少的标签)；而某些标签，比如颜色(color)，模型就不能达到较优的性能(弱标注，weakly labelled). 总体而言，模型对于大部分标签取得了超过 95% 的精度.3.1. 数据加载与处理实现接 2.1.# 选取部分数据，进一步分析 analysis_df = df.sample(frac=0.95, random_state=10) analysis_df.reset_index(drop=True, inplace=True) labels = analysis_df.keys()[1:-1].values N = len(analysis_df) print('Total nuber of Data_points {}\nLabels {}'.format(N, labels)) ''' Total nuber of Data_points 10269 Labels ['gender' 'subCategory' 'articleType' 'baseColour' 'season' 'usage'] ''' # 可视化，图片及其标签. randm = np.random.randint(0, N) img = mpimg.imread(analysis_df['img_path'][randm]) plt.figure(figsize=(7,7)) plt.imshow(img) plt.title(str(analysis_df[labels].values[randm])) plt.xlabel('Product_id :: {} \n Image shape :: {}'.format( str(analysis_df['id'][randm]), img.shape)) plt.show()# 图片加载及预处理函数 def load_image(path, shape=(112,112,3)): image_list = np.zeros((len(path), shape[0], shape[1], shape[2])) for i, fig in enumerate(path): img = image.load_img(fig, color_mode='rgb', target_size=shape) x = image.img_to_array(img).astype('float16') x = x / 255.0 image_list[i] = x return image_list # 标签加载与处理 def load_attr(df, attr, N=None, det=False, one_hot=True): le = LabelEncoder() le.fit(df[attr]) target = le.transform(df[attr]) if N is None: N = len(set(target)) if one_hot: target = to_categorical(target, num_classes=N) if det: print('\n{}:: \n{}'.format(attr,le.classes_)) print('Target shape', target.shape) return le.classes_, N, target # 数据分片 # 训练数据 x_train = load_image(analysis_df['img_path'][:t]) gen_names, num_gen, gender_tr = load_attr(analysis_df[:t], 'gender', det=True) sub_names, num_sub, subCategory_tr = load_attr(analysis_df[:t], 'subCategory', det=True) art_names, num_art, articleType_tr = load_attr(analysis_df[:t], 'articleType', det=True) col_names, num_col, baseColour_tr = load_attr(analysis_df[:t], 'baseColour', det=True) sea_names, num_sea, season_tr = load_attr(analysis_df[:t], 'season', det=True) use_names, num_use, usage_tr = load_attr(analysis_df[:t], 'usage', det=True) # 验证数据 x_val = load_image(analysis_df['img_path'][t:-100]) _, _, gender_val = load_attr(analysis_df[t:-100], 'gender', N=num_gen) _, _, subCategory_val = load_attr(analysis_df[t:-100], 'subCategory', N=num_sub) _, _, articleType_val = load_attr(analysis_df[t:-100], 'articleType', N=num_art) _, _, baseColour_val = load_attr(analysis_df[t:-100], 'baseColour', N=num_col) _, _, season_val = load_attr(analysis_df[t:-100], 'season', N=num_sea) _, _, usage_val = load_attr(analysis_df[t:-100], 'usage', N=num_use) # 后 100 张图片作为测试集 dict_ = {'gen_names' : gen_names.tolist(), 'sub_names' : sub_names.tolist(), 'art_names' : art_names.tolist(), 'col_names' : col_names.tolist(), 'sea_names' : sea_names.tolist(), 'use_names' : use_names.tolist()} json.dump(dict_, open('label_map.json', 'w')) print('\n Distinct classes (Per label):',num_gen, num_sub, num_art, num_col, num_sea, num_use) print('Shape:: Train: {}, Val: {}'.format(x_train.shape, x_val.shape))输出如：gender:: ['Boys' 'Men' 'Women'] Target shape (9242, 3) subCategory:: ['Innerwear' 'Topwear'] Target shape (9242, 2) articleType:: ['Briefs' 'Kurtas' 'Shirts' 'Tops' 'Tshirts'] Target shape (9242, 5) baseColour:: ['Black' 'Blue' 'Green' 'Grey' 'Navy Blue' 'Purple' 'Red' 'White'] Target shape (9242, 8) season:: ['Fall' 'Summer'] Target shape (9242, 2) usage:: ['Casual' 'Ethnic' 'Formal' 'Sports'] Target shape (9242, 4) Distinct classes (Per label): 3 2 5 8 2 4 Shape:: Train: (9242, 112, 112, 3), Val: (927, 112, 112, 3)3.2. 模型结构class Classifier(): ''' Contains Multi-label Multi-class architecture ***Arguments*** input_shape :: Input shape, format : (img_rows, img_cols, channels), type='tuple' pre_model :: Pretrained model file path, file extension : '.h5', type='str' ***Fuctions*** build_model() :: Define the CNN model (can be modified & tuned as per usecase) train_model() :: Model Training (optimizers and metrics can be modified) eval_model() :: Predict classes (in one-hot encoding) ''' def __init__(self, input_shape=(112,112,3), pre_model=None): self.img_shape = input_shape # Load/Build Model if pre_model is None: self.cnn_model = self.build_model() else: try: self.cnn_model = load_model(pre_model) except OSError: print('Unable to load {}'.format(pre_model)) # Compile Model self.cnn_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) plot_model(self.cnn_model, to_file='consolidated_model.png') self.cnn_model.summary() def custom_residual_block(self, r, channels): r = BatchNormalization()(r) r = Conv2D(channels, (3, 3), activation='relu')(r) h = Conv2D(channels, (3, 3), padding='same', activation='relu')(r) h = Conv2D(channels, (3, 3), padding='same', activation='relu')(h) h = Conv2D(channels, (3, 3), padding='same', activation='relu')(h) return add([h, r]) def build_model(self): input_layer = Input(shape=self.img_shape) # Conv Layers en_in = Conv2D(16, (5, 5), activation='relu')(input_layer) h = MaxPooling2D((3, 3))(en_in) h = self.custom_residual_block(h, 32) h = self.custom_residual_block(h, 32) h = self.custom_residual_block(h, 48) h = self.custom_residual_block(h, 48) h = self.custom_residual_block(h, 48) h = self.custom_residual_block(h, 48) h = self.custom_residual_block(h, 54) h = self.custom_residual_block(h, 54) en_out = self.custom_residual_block(h, 54) # Dense gender gen = self.custom_residual_block(en_out, 48) gen = GlobalAveragePooling2D()(gen) gen = Dense(100, activation='relu')(gen) gen = Dropout(0.1)(gen) gen = Dense(50, activation='relu')(gen) gen_out = Dense(num_gen, activation='softmax', name= 'gen_out')(gen) # Dense subCategory sub = self.custom_residual_block(en_out, 48) sub = GlobalAveragePooling2D()(sub) sub = Dense(100, activation='relu')(sub) sub = Dropout(0.1)(sub) sub = Dense(50, activation='relu')(sub) sub_out = Dense(num_sub, activation='softmax', name= 'sub_out')(sub) # Dense articleType art = self.custom_residual_block(en_out, 48) art = GlobalAveragePooling2D()(art) art = Dense(100, activation='relu')(art) art = Dropout(0.1)(art) art = Dense(50, activation='relu')(art) art_out = Dense(num_art, activation='softmax', name= 'art_out')(art) # Dense baseColour col = self.custom_residual_block(en_out, 48) col = GlobalAveragePooling2D()(col) col = Dense(100, activation='relu')(col) col = Dropout(0.1)(col) col = Dense(50, activation='relu')(col) col_out = Dense(num_col, activation='softmax', name= 'col_out')(col) # Dense season sea = self.custom_residual_block(en_out, 48) sea = GlobalAveragePooling2D()(sea) sea = Dense(100, activation='relu')(sea) sea = Dropout(0.1)(sea) sea = Dense(50, activation='relu')(sea) sea_out = Dense(num_sea, activation='softmax', name= 'sea_out')(sea) # Dense usage use = self.custom_residual_block(en_out, 48) use = GlobalAveragePooling2D()(use) use = Dense(100, activation='relu')(use) use = Dropout(0.1)(use) use = Dense(50, activation='relu')(use) use_out = Dense(num_use, activation='softmax', name='use_out')(use) return Model(input_layer, [gen_out, sub_out, art_out, col_out, sea_out, use_out]) def train_model(self, x_train, y_train, x_val, y_val, save_path, epochs=25, batch_size=16): # early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto', restore_best_weights=True) # check_pointer = ModelCheckpoint(save_path, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=1) # history = self.cnn_model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_val, y_val), callbacks=[early_stopping, check_pointer]) return history def eval_model(self, x_test): preds = self.cnn_model.predict(x_test) return preds模型结构输出部分如：3.3. 模型训练ce = Classifier(pre_model=None) history = ce.train_model(x_train, [gender_tr, subCategory_tr, articleType_tr, baseColour_tr, season_tr, usage_tr], x_val, [gender_val, subCategory_val, articleType_val, baseColour_val, season_val, usage_val], save_path='consolidated_model.h5', epochs=25, batch_size=30)3.4. 训练曲线可视化 def learning_curves(hist): ''' Learing curves included (losses and accuracies per label) ''' plt.style.use('ggplot') epochs = range(1,len(hist['loss'])+1) colors = ['b', 'g' ,'r', 'c', 'm', 'y'] loss_ = [s for s in hist.keys() if 'loss' in s and 'val' not in s] val_loss_ = [s for s in hist.keys() if 'loss' in s and 'val' in s] acc_ = [s for s in hist.keys() if 'acc' in s and 'val' not in s] val_acc_ = [s for s in hist.keys() if 'acc' in s and 'val' in s] # Loss (per label) plt.figure(1, figsize=(20,10)) for tr, val ,c in zip(loss_, val_loss_, colors): plt.plot(epochs, hist[tr], c, label='train_{} : {}'.format(tr, str(format(hist[tr][-1],'.3f')))) plt.plot(epochs, hist[val], '-.'+c, label='{} : {}'.format(val, str(format(hist[val][-1],'.3f')))) plt.title('Model Loss (Distinct labels)') plt.xlabel('Epochs') plt.ylabel('Loss') plt.legend(loc='upper right') plt.show() # Accuracy (per label) plt.figure(2, figsize=(20,10)) for tr, val, c in zip(acc_, val_acc_, colors): plt.plot(epochs, hist[tr], c, label='train_{} : {}'.format(tr, str(format(hist[tr][-1],'.3f')))) plt.plot(epochs, hist[val], '-.'+c, label='{} : {}'.format(val, str(format(hist[val][-1],'.3f')))) plt.title('Model Accuracy (Distinct labels)') plt.xlabel('Epochs') plt.ylabel('Accuracy') plt.legend(loc='lower right') plt.show() learning_curves(history.history)3.5. 预测模型训练完成后，即可加载示例图片，并预测标签输出.class Recognize_Item(object): ''' Required: image_path: path or url of the image file [Required] model file: to be saved/updated at consolidated_model.h5 label_dict: to be saved/updated at label_map.json ''' def __init__(self, model_path='consolidated_model.h5'): self.model_path = model_path def load_model(self): ''' Load Consolidated model from utils ''' try: item_reco_model = load_model(self.model_path) return item_reco_model except Exception: raise Exception('#-----Failed to load model file-----#') def process_img(self, image_path): ''' image: load & preprocess image file ''' img = image.load_img(image_path, color_mode='rgb', target_size=(112,112,3)) x = image.img_to_array(img).astype('float32') return x / 255.0 def tmp_fn(self, one_hot_labels): ''' tmp function to manpulate all one-hot encoded values labels to -- -- a vector of predicted categories per labels ''' flatten_labels = [] for i in range(len(one_hot_labels)): flatten_labels.append(np.argmax(one_hot_labels[i], axis=-1)[0]) return self.class_map(flatten_labels) def class_map(self, e): ''' To convert class encoded values to actual label attributes ''' label_map = json.load(open('label_map.json')) dict_ = {'gen_names' : gen_names.tolist(), 'sub_names' : sub_names.tolist(), 'art_names' : art_names.tolist(), 'col_names' : col_names.tolist(), 'sea_names' : sea_names.tolist(), 'use_names' : use_names.tolist()} gender = label_map['gen_names'][e[0]] subCategory = label_map['sub_names'][e[1]] articleType = label_map['art_names'][e[2]] baseColour = label_map['col_names'][e[3]] season = label_map['sea_names'][e[4]] usage = label_map['use_names'][e[5]] # return [gender, subCategory, articleType, baseColour, season, usage] def predict_all(self, model, image): ''' Predict all labels ''' x_image = np.expand_dims(image, axis=0) return self.tmp_fn(model.predict(x_image)) def demo_results(self, image_path, ground_truth=None): ''' To demo sample predictions ''' image = self.process_img(image_path) predicted_labels = self.predict_all(model, image) # plt.figure(figsize=(5,7)) plt.title('Fashion: Clothing Item recognition module') plt.text(0, 133, str('Predictions : '+' | '.join(predicted_labels)), fontsize=12, color='teal') # if ground_truth is None: pass else: plt.text(0, 142, str('Ground Truth : '+' | '.join(ground_truth)), fontsize=12, color='b') # plt.axis('off') plt.imshow(image) plt.show() # reco = Recognize_Item() global model model = reco.load_model() # 预测和GT. test_id = N-np.random.randint(0, 100) reco.demo_results(analysis_df['img_path'][test_id], ground_truth=analysis_df.ix[test_id,'gender':'usage'],) # 随机测试 test_id = N-np.random.randint(0, 100) reco.demo_results(analysis_df['img_path'][test_id])4. AWS 部署可参考原文，基于 Flask、Docker、AWS 的部署.