Github - PyDenseCRF

Efficient Inference in Fully Connected CRFs with Gaussian Edge Potentials - NIPS2011

全连接 CRFs，dense CRFs 的 Cython-based Python 封装.

1. 安装

Ubuntu 中可以采用 pip 安装：

sudo pip install pydensecrf

这里需要采用 Cython 较新版本，比如 Cython>=0.22.

2. 简单使用

对于图像，pydensecrf 库的最简单的使用方式是 DenseCRF2D 类.

import numpy as np
import pydensecrf.densecrf as dcrf
    
d = dcrf.DenseCRF2D(640, 480, 5)  # width, height, nlabels

2.1. 一元潜在关系 Unary potential

设置固定的一元关系(Unary potential):

U = np.array(...)     # Get the unary in some way.
print(U.shape)        # -> (5, 480, 640)
print(U.dtype)        # -> dtype('float32')
U = U.reshape((5,-1)) # Needs to be flat.
d.setUnaryEnergy(U)

# Or alternatively: d.setUnary(ConstUnary(U))

其中，U 是 negative log-probabilites. 如果采用的概率为 py，则需要先进行 U=-np.log(py) 处理. nlabels 维应该在 reshape 前.

print(U.shape)  # -> (480, 640, 5)
U = U.transpose(2, 0, 1).reshape((5,-1))

Unary potential 的设定：

  • [1] - 手工或其它方式生成的 hard labeling. from pydensecrf.utils import unary_from_labels
  • [2] - 计算的概率分布，如深度网络的 softmax 输出. from pydensecrf.utils import unary_from_softmax

2.2. 成对多元关系 Pairwise potentials

常用的成对关系添加：

# This adds the color-independent term, features are the locations only.
d.addPairwiseGaussian(sxy=(3,3), 
                      compat=3, 
                      kernel=dcrf.DIAG_KERNEL, 
                      normalization=dcrf.NORMALIZE_SYMMETRIC)

# This adds the color-dependent term, i.e. features are (x,y,r,g,b).
# im is an image-array, e.g. im.dtype == np.uint8 and im.shape == (640,480,3)
d.addPairwiseBilateral(sxy=(80,80), 
                       srgb=(13,13,13), 
                       rgbim=im, 
                       compat=10, 
                       kernel=dcrf.DIAG_KERNEL,
                       normalization=dcrf.NORMALIZE_SYMMETRIC)

对应的简化形式：

d.addPairwiseGaussian(sxy=3, compat=3)
d.addPairwiseBilateral(sxy=80, srgb=13, rgbim=im, compat=10)

3. Demo.py

3.1. dense_crf 函数

dense_crf 函数定义：

import numpy as np
import pydensecrf.densecrf as dcrf

def dense_crf(img, output_probs):
    h = output_probs.shape[0]
    w = output_probs.shape[1]

    output_probs = np.expand_dims(output_probs, 0)
    output_probs = np.append(1 - output_probs, output_probs, axis=0)

    d = dcrf.DenseCRF2D(w, h, 2)
    U = -np.log(output_probs)
    U = U.reshape((2, -1))
    U = np.ascontiguousarray(U)
    img = np.ascontiguousarray(img)
    
    U = U.astype(np.float32)
    d.setUnaryEnergy(U) # Unary 

    d.addPairwiseGaussian(sxy=20, compat=3)  # 
    d.addPairwiseBilateral(sxy=30, srgb=20, rgbim=img, compat=10)

    Q = d.inference(5)
    Q = np.argmax(np.array(Q), axis=0).reshape((h, w))

    return Q

参考 - pydensecrf - exmaple/inference.py

3.2. 测试 demo

#image - 原始图片，hxwx3，采用 PIL.Image 读取
#seg_map - 假设为语义分割的 mask, hxw, np.array 形式.

import numpy as np
import matplotlib.pyplot as plt

final_mask = dense_crf(np.array(image).astype(np.uint8), seg_map)
plt.subplot(1, 3, 1)
plt.imshow(image)
plt.subplot(1, 3, 2)
plt.imshow(seg_map)
plt.subplot(1, 3, 3)
plt.imshow(final_mask)
plt.show()

4. Github使用 - tensorflow-deeplab-resnet/inference_crf.py

基于 Tensorflow 实现采用 CNNs 和 CRFs 的语义分割

    #---------------------------------CRF
    import sys

    import pydensecrf.densecrf as dcrf
    from pydensecrf.utils import compute_unary, create_pairwise_bilateral, \
        create_pairwise_gaussian, softmax_to_unary

    import skimage.io as io

    softmax = processed_probabilities.transpose((2, 0, 1))  # processed_probabilities：CNN 预测概率

    # softmax_to_unary 的输入是概率值的负对数
    unary = softmax_to_unary(softmax)
    # The inputs should be C-continious -- we are using Cython wrapper
    unary = np.ascontiguousarray(unary)  #(21, n)
    d = dcrf.DenseCRF(img.shape[0] * img.shape[1], 21)

    d.setUnaryEnergy(unary)

    # This potential penalizes small pieces of segmentation that are
    # spatially isolated -- enforces more spatially consistent segmentations
    feats = create_pairwise_gaussian(sdims=(10, 10), shape=img.shape[:2])

    d.addPairwiseEnergy(feats, compat=3,
                        kernel=dcrf.DIAG_KERNEL,
                        normalization=dcrf.NORMALIZE_SYMMETRIC)

    # This creates the color-dependent features --
    # because the segmentation that we get from CNN are too coarse
    # and we can use local color features to refine them
    feats = create_pairwise_bilateral(sdims=(50, 50), schan=(20, 20, 20),
                                      img=img, chdim=2)

    d.addPairwiseEnergy(feats, compat=10,
                        kernel=dcrf.DIAG_KERNEL,
                        normalization=dcrf.NORMALIZE_SYMMETRIC)
    #迭代次数,对于IMG_1702(2592*1456)这张图,迭代5 16.807087183s 迭代20 37.5700438023s
    Q = d.inference(5)
    res = np.argmax(Q, axis=0).reshape((img.shape[0], img.shape[1]))

5. Github使用 - carvana-challenge/util/crf.py

import time
import numpy as np
import torch
from torch.autograd import Variable

import pydensecrf.densecrf as dcrf
from pydensecrf.utils import compute_unary, create_pairwise_bilateral, create_pairwise_gaussian, unary_from_softmax

def run_crf(masks):
    print("You're using CRF. Are you sure? In our previous experiments, it has never improved the performance. ")
    crf_masks = np.zeros(masks.data.size())  # shape: (batch_size, 1, height, width)
    for img_idx in range(len(img_name)):
        img  =  images.data[img_idx].cpu().numpy()  # shape: (3, height, width)
        prob = outputs.data[img_idx].cpu().numpy()  # shape: (1, height, width)
        crf_masks[img_idx] = crf(img, prob)

        # convert CRF results back into Variable in GPU
        masks = Variable(torch.from_numpy(crf_masks).float(), volatile=True)
        if torch.cuda.is_available():
            masks = masks.cuda()
    return masks

def crf(img, prob):
    '''
    input:
      img: numpy array of shape (num of channels, height, width)
      prob: numpy array of shape (1, height, width), neural network last layer sigmoid output for img

    output:
      res: (1, height, width)

    Modified from:
      http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/18/image-segmentation-with-tensorflow-using-cnns-and-conditional-random-fields/
      https://github.com/yt605155624/tensorflow-deeplab-resnet/blob/e81482d7bb1ae674f07eae32b0953fe09ff1c9d1/inference_crf.py
    '''
    func_start = time.time()

    img = np.swapaxes(img, 0, 2)
    # img.shape: (width, height, num of channels)

    num_iter = 5

    prob = np.swapaxes(prob, 1, 2)  # shape: (1, width, height)

    # preprocess prob to (num_classes, width, height) since we have 2 classes: car and background.
    num_classes = 2
    probs = np.tile(prob, (num_classes, 1, 1))  # shape: (2, width, height)
    probs[0] = np.subtract(1, prob) # class 0 is background
    probs[1] = prob                 # class 1 is car

    d = dcrf.DenseCRF(img.shape[0] * img.shape[1], num_classes)

    unary = unary_from_softmax(probs)  # shape: (num_classes, width * height)
    unary = np.ascontiguousarray(unary)
    d.setUnaryEnergy(unary)

    # This potential penalizes small pieces of segmentation that are
    # spatially isolated -- enforces more spatially consistent segmentations
    feats = create_pairwise_gaussian(sdims=(10, 10), shape=img.shape[:2])
    d.addPairwiseEnergy(feats, compat=3,
                        kernel=dcrf.DIAG_KERNEL,
                        normalization=dcrf.NORMALIZE_SYMMETRIC)
    # Note that this potential is not dependent on the image itself.

    # This creates the color-dependent features --
    # because the segmentation that we get from CNN are too coarse
    # and we can use local color features to refine them
    feats = create_pairwise_bilateral(sdims=(50, 50), schan=(20, 20, 20),
                                      img=img, chdim=2)

    d.addPairwiseEnergy(feats, compat=10,
                        kernel=dcrf.DIAG_KERNEL,
                        normalization=dcrf.NORMALIZE_SYMMETRIC)


    Q = d.inference(num_iter)  # set the number of iterations
    res = np.argmax(Q, axis=0).reshape((img.shape[0], img.shape[1]))
    # res.shape: (width, height)

    res = np.swapaxes(res, 0, 1)  # res.shape:    (height, width)
    res = res[np.newaxis, :, :]   # res.shape: (1, height, width)

    func_end = time.time()
    # print('{:.2f} sec spent on CRF with {} iterations'.format(func_end - func_start, num_iter))
    # about 2 sec for a 1280 * 960 image with 5 iterations
    return res
Last modification：May 13, 2019
© Allow specification reprint
If you think my article is useful to you, please feel free to appreciate