\documentclass[10pt,journal,cspaper,compsoc]{IEEEtran}
\usepackage{cite}
\usepackage{algorithmic}
\usepackage{array}
\usepackage{mdwmath}
\usepackage{mdwtab}
\usepackage{eqparbox}
\usepackage[tight,footnotesize]{subfigure}
\usepackage{cite}
\usepackage{url}
\usepackage{multicol,multirow}
\usepackage[cmex10]{amsmath}
\usepackage{makeidx}
\makeindex

%\usepackage[pdftex,draft]{graphicx}
\usepackage[pdftex]{graphicx}
\DeclareGraphicsExtensions{.pdf,.jpg,.png}

\usepackage[pdfborder=false,colorlinks,bookmarksnumbered,bookmarksopen,linkcolor=black,citecolor=black,urlcolor=black]{hyperref}
\usepackage{overpic}
\usepackage{rotating,soul}
\usepackage[usenames,dvipsnames]{color}

\renewcommand{\tablename}{Table }

\newcommand{\figref}[1]{Fig.~\ref{#1}}
\newcommand{\tabref}[1]{Fig.~\ref{#1}}
\newcommand{\equref}[1]{(\ref{#1})}
\newcommand{\secref}[1]{Sec.~\ref{#1}}
\newcommand{\cmm}[1]{{\textcolor{black}{#1}}}
\newcommand{\reviewer}[1]{{\textcolor{black}{#1}}}
\newcommand{\todo}[1]{{\textcolor{red}{\bf ToDo: #1}}}
\newcommand{\mcols}[1]{\multicolumn{2}{c}{#1}}
\newcommand{\Cols}[1]{\multicolumn{2}{c|}{#1}}
\newcommand{\Rows}[1]{\multirow{2}{*}{#1}}
\newcommand{\colInf}{T50 & T100}
\newcommand{\ColInf}{\small{Flickr} & \small{Our} & \small{SHoG}}
\newcommand{\myPara}[1]{\noindent\textbf{#1.}}

\newcommand{\IT}{IT\cite{98pami/Itti}}
\newcommand{\MZ}{MZ\cite{03ACMMM/Ma_Contrast-based}}
\newcommand{\GB}{GB\cite{07ANIPS/harel_graph}}
\newcommand{\SR}{SR\cite{07cvpr/hou_SpectralResidual}}
\newcommand{\FT}{FT\cite{09cvpr/Achanta_FTSaliency}}
\newcommand{\CA}{CA\cite{10cvpr/goferman_context}}
\newcommand{\LC}{LC\cite{06acmmm/ZhaiS_spatiotemporal}}
\newcommand{\AC}{AC\cite{08cvs/achanta_salient}}
\newcommand{\AIM}{AIM\cite{09jov/bruce_saliency}}
\newcommand{\IM}{IM\cite{11cvpr/Murray_Saliency}}
\newcommand{\MSS}{MSS\cite{10icip/Achanta_saliency}}
\newcommand{\SEG}{SEG\cite{10eccv/rahtu_segmenting}}
\newcommand{\SeR}{SeR\cite{09jov/seo_static}}
\newcommand{\SUN}{SUN\cite{08jov/zhang-sun}}
\newcommand{\LD}{LD\cite{10pami/Liu_Learning}}
\newcommand{\SWD}{SWD\cite{11cvpr/swd}}
\newcommand{\SHoG}{SHoG\cite{11tvcg/eitz_sbir}}
\newcommand{\CB}{CB\cite{11bmvc/Jiang_Saliency}}
\newcommand{\LP}{LP\cite{Judd_2009}}
\newcommand{\NumMethods}{15~}

\newcommand{\TSpace}{\vspace{-.05in}}
\newcommand{\BSpace}{\vspace{-.05in}}

\newcommand{\HC}{HC-maps }
\newcommand{\RC}{RC-maps }
\newcommand{\Lab}{$L^*a^*b^*$}
\newcommand{\GRAB}{\cite{04tog/rother_grabcut}}

\newcommand{\PutOverDes}{\small \put(13,0){Gau} \put(29,0){FT}\put(42,0){SEG}\put(56,0){\GRAB}\put(75,0){CB}\put(88,0){Ours}}

\graphicspath{{./Imgs/}}

\hyphenation{op-tical net-works semi-conduc-tor} % correct bad hyphenation here


\begin{document}


\title{Salient Object Detection and Segmentation}
\title{Global Contrast based Salient Region Detection}


\author{Ming-Ming~Cheng, Niloy~J.~Mitra, Xiaolei~Huang, %~\IEEEmembership{Member,~IEEE,} \\
        Philip~H.~S.~Torr, %~\IEEEmembership{Senior Member,~IEEE,}
        and~Shi-Min~Hu %,~\IEEEmembership{Member,~IEEE}% <-this % stops a space
    \IEEEcompsocitemizethanks{
        \IEEEcompsocthanksitem M.M. Cheng is with Nankai University.
            %M.M. Cheng is with Oxford University.
            N.J. Mitra is with UCL. X. Huang is with Lehigh University.
            P.H.S. Torr is with Oxford University.
            Shi-Min Hu (corresponding author) is with TNList, Tsinghua University.
        \IEEEcompsocthanksitem A preliminary version of this work appeared at CVPR
            \cite{11cvpr/Cheng_Saliency}.
            The C++ source code, benchmark dataset, results,
            highlighted applications and FAQs are available via our project page:
            \href{http://mmcheng.net/SalObj/}{http://mmcheng.net/SalObj/}.
            The major part of this work was done in Tsinghua University.
    }% <-this % stops a space   %    \thanks{}
}


\markboth{IEEE TRANSACTIONS ON PATTERN ANALYSIS AND MACHINE INTELLIGENCE,~Vol.~xx,No.~xx,~xxx.~xxxx}%
{Cheng \MakeLowercase{\textit{et al.}}: Global Contrast based Salient Region Detection}
%\pubid{0000--0000/00\$00.00~\copyright~2009 IEEE}

\IEEEcompsoctitleabstractindextext{%
    \begin{abstract}
        Automatic estimation of salient object regions across images,
        without any prior assumption or knowledge of the contents of the corresponding scenes,
        enhances many computer vision and computer graphics applications.
        We introduce a regional contrast based salient object detection algorithm,
        which simultaneously evaluates global contrast differences
        and spatial weighted coherence scores.
        The proposed algorithm is simple, efficient, naturally multi-scale,
        and produces full-resolution, high-quality saliency maps.
        These saliency maps are further used to initialize a novel iterative
        version of GrabCut, namely SaliencyCut,
        for high quality unsupervised salient object segmentation.
        We extensively evaluated our algorithm using traditional salient
        object detection datasets,
        as well as a more challenging Internet image dataset.
        Our experimental results demonstrate that our algorithm consistently
        outperforms 15 existing salient object detection and segmentation methods,
        yielding higher precision and better recall rates.
        We also show that our algorithm can be used to efficiently extract
        salient object masks from Internet images,
        enabling effective sketch-based image retrieval (SBIR) via simple shape comparisons.
        Despite such noisy internet images, where the saliency regions are ambiguous,
        our saliency guided image retrieval achieves a superior retrieval rate
        compared with state-of-the-art SBIR methods,
        and additionally provides important target object region information.
    \end{abstract}

    \begin{keywords}
    Salient object detection, visual attention, saliency map, unsupervised segmentation, image retrieval.
    \end{keywords}
}


\maketitle

\IEEEdisplaynotcompsoctitleabstractindextext
\IEEEpeerreviewmaketitle


\section{Introduction}\label{sec:Introduction}

\IEEEPARstart{W}{e}, as humans, are experts at quickly and accurately identifying
the most visually noticeable  foreground object in the scene, known as salient objects,
and adaptively focus our attention on such perceived important regions.
In contrast, computationally identifying such salient object regions
\cite{10pami/Liu_Learning,borjisalient}, that match the human annotators'
behaviour when they have been asked to pick a salient object in an image,
is very challenging.
Being able to automatically, efficiently, and accurately estimate salient object regions,
however, is highly desirable given the immediate ability to
characterise the spatial support for feature extraction,
isolate the object from potentially confusing background,
and preferentially allocate finite computational resources for subsequent image processing.

While essentially solving a segmentation problem,
salient object detection approaches segment only the salient foreground object from the background,
rather than partition an image into regions of coherent properties
as in general segmentation algorithms~\cite{borjisalient}.
Salient object detection models also differ from eye fixation prediction models
that predict a few fixation points in an image
rather than uniformly highlighting the entire salient object region \cite{borjisalient}.
In practice, salient object detection methods are commonly used as a first step of many graphics/vision applications including
%
object-of-interest image segmentation
\cite{09iccv/donoser_saliency}, % 06TCSVT/han_unsupervised,06josa/KoN_InterestSegmentation,
object recognition~\cite{04cvpr/RutishauserWWKP}, % ,fraundorfer2003detecting,carreira2012cpmc,alexe2012measuring
adaptive compression of images~\cite{00CE/christopoulos_jpeg},
content-aware image editing~\cite{09cgf/ZhangC,ChengRepFinder10},
 image retrieval \cite{tog09/ChenCT_Sketch2Photo,hu2013internet,gao2013visual}, etc.



\begin{figure}[t!]
    \centering
    \includegraphics[width=\columnwidth]{teaserN.pdf} \\
    \TSpace
    \caption{Given an input image~(top), a global contrast analysis is used
        to compute a high resolution saliency map~(middle), which can be used to
        produce an unsupervised segmentation mask~(bottom) for an object of interest.
    }\label{fig:teaser}
    \BSpace
\end{figure}

Although extraction of salient objects in a scene is related to
accurate image segmentation and object retrieval, interestingly,
reliable saliency estimation is often feasible {\em without} any actual scene understanding.
Saliency, as widely believed, is a bottom-up process that originates from visual distinctness,
rarity, or surprise and is often attributed to variations in image attributes such as color,
gradient, edges, and boundaries \cite{borji2012state}.
%
%This is feasible since often saliency, as widely believed, is bottom-up.
%Such a hypothesis is favored by an evolutionary incentive to parallel
%%Review 1 do not like this word
%process large volumes of low-level image cues,
%without requiring any computationally expensive global coupling. %%Review 1 do not like this word
%Hence, saliency often originates from visual uniqueness, unpredictability, rarity, or surprise,
%and is often attributed to variations in image attributes like color,
%gradient, edges, and boundaries.
%
Visual saliency, being closely related to our perception and processing of visual stimuli,
is investigated across many disciplines including cognitive
psychology~\cite{55ARP/Teuber_physiological,04nature/Wolfe_attributesVisual},
neurobiology~\cite{95ARN/DesimoneNeuralMachanisms,09biology/eyeMovement}, and computer
vision~\cite{98pami/Itti,13iccv/Cheng_Saliency,hou2012image}.
%
Based on our observed reaction times and estimated signal transmission
times along biological pathways,
human attention theories  hypothesize
that the human vision system processes only parts of an image in detail,
while leaving the rest nearly unprocessed.
%
Early work by Treisman and Gelade~\cite{80cogSc/Treisman_featureIntegration}, Koch and
Ullman~\cite{85HN/KochVisualAttention}, and subsequent attention theories proposed by Itti, Wolfe
and others, suggest two stages of visual attention: a fast, pre-attentive, bottom-up, data driven
saliency extraction; followed by a slower, task dependent, top-down, goal driven saliency extraction.




\begin{figure*}[t!]
    \subfigure[original]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895}}
    \subfigure[\IT]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_IT}}
    \subfigure[\AIM]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_AIM}}
    \subfigure[\IM]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_IM}}
    \subfigure[\MSS]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_MSS}}
    \subfigure[\SEG]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_SEG}}
    \subfigure[\SeR]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_SeR}}
    \subfigure[\SUN]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_SUN}}
    \subfigure[\SWD]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_SWD}} \vspace{-.1in}
    \\
    \subfigure[\GB]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_GB}}
    \subfigure[\SR]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_SR}}
    \subfigure[\AC]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_AC}}
    \subfigure[\CA]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_CA}}
    \subfigure[\FT]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_FT}}
    \subfigure[\LC]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_LC}}
    \subfigure[\CB]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_CB}}
    \subfigure[HC]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_HC}}
    \subfigure[RC]{\includegraphics[height=.1045\textwidth]{all_comparison/30_30895_RC}}\\
    \vspace{-.15in}
    \caption{Saliency maps computed by different state-of-the-art methods~(b-p),
      and with our proposed HC~(q) and RC methods~(r). Most results
      highlight edges, or are of low resolution.
      See also \figref{fig:VisualComparison} and our project webpage.
   }\label{fig:cmp1vAll}
   \BSpace
\end{figure*}


We focus on bottom-up data driven salient object detection using image contrast
(see \figref{fig:teaser}) under the assumption that a salient object exists in an image \cite{10pami/Liu_Learning}.
The proposed method is simple, fast, and produces high quality results on benchmark datasets.
%
Motivated by the popular belief that human cortical cells may be \emph{hard wired}
to preferentially respond to high contrast stimulus in their receptive fields
\cite{03neuron/Reynolds_attentionSaliency},
%
we propose contrast analysis for extracting high-resolution,
full-field saliency maps based on the following considerations:
\begin{itemize}
  \item A global contrast based method, which separates a large-scale object from its surroundings,
    is desirable over local contrast based methods producing high saliency values at or near object boundaries.
    Global considerations enable assignment of comparable saliency values across similar image regions,
    and can uniformly highlight entire objects.
  \item Saliency of a region mainly depends on its contrast with respect to its nearby regions,
    while contrasts to distant regions are less significant (see also \cite{99reinagel_natural}).
  \item In man-made photographs, object are often concentrated towards the inner regions of the images,
    away from image boundaries (see \cite{11bmvc/Jiang_Saliency}).
  \item Saliency maps should be fast, accurate, have low memory footprints, and easy to generate to allow processing of
    large image collections, and facilitate efficient image classification and retrieval.
\end{itemize}


We propose a \emph{histogram-based contrast method}~(HC) to measure saliency.
%
\HC assign pixel-wise saliency values based simply on color separation from all other image
pixels to produce full resolution saliency maps. We use a histogram-based approach for efficient processing,
while employing a smoothing procedure to reduce quantization artifacts.
%
%Note that our algorithm is targeted towards natural scenes,
%and maybe suboptimal for extracting saliency of highly textured scenes
%(see \figref{fig:challenging_maps}).
%Highly textured scenes is also a small part of natural scenes.
%It's not contrast to natural scenes. Maybe change a representation?}.


As an improvement over HC-maps, we incorporate spatial relations to produce
\emph{region-based contrast}~(RC) maps where we first segment the input image into
regions, and then assign saliency values to them. The saliency value of a region is
then calculated using a global contrast score, measured by the region's
contrast and spatial distances to other regions in the image.
Note that this approach better acknowledges the relation between
image segmentation and saliency determination.



Segmenting regions of interest in still images is of great practical importance in
many computer vision and graphics applications.
Researchers have devoted significant efforts to minimize
user interaction during this process.
GrabCut~\cite{04tog/rother_grabcut}, which iteratively optimizes the energy function
and considers both texture and edge information,
has successfully simplified the user interaction to
simply dragging a rectangle around the desired object.
We propose SaliencyCut,
an improved iterative version of GrabCut,
and combine it with our saliency detection method
to achieve superior performance compared to
state-of-the-art unsupervised salient object extraction methods.



In order to evaluate the proposed algorithms and compare with state-of-the-art alternatives, we build a database with 10,000 pixel-accurate human-labeled ground truth images (see also \secref{sec:MSRA10K}),
which is an order of magnitude bigger than previous largest
public available dataset of its kind \cite{09cvpr/Achanta_FTSaliency}.
We have extensively evaluated our methods on this dataset, and compared
our methods with 15 state-of-the-art saliency methods
%~\cite{98pami/Itti,06acmmm/ZhaiS_spatiotemporal,07ANIPS/harel_graph,07cvpr/hou_SpectralResidual,08cvs/achanta_salient,08jov/zhang-sun,09jov/seo_static,09cvpr/Achanta_FTSaliency,09jov/bruce_saliency,10cvpr/goferman_context,10icip/Achanta_saliency,10eccv/rahtu_segmenting,11cvpr/Murray_Saliency,11cvpr/swd,11bmvc/Jiang_Saliency}
as well as with manually created ground truth annotations.
%
The experiments show significant improvements over previous methods both in terms of precision and recall rates.
%
Overall, compared with HC-maps,
\RC produce better precision and recall rates,
but at the cost of increased computational overheard.
In our extensive empirical evaluations,
we observe that the unsupervised segmentation results produced by
our SaliencyCut method are, in most cases, are
comparable to the manually annotated ground truths.
%
We also demonstrate applications of the extracted saliency maps to segmentation and
sketch-based image retrieval. %, context-aware resizing, and non-photo realistic rendering.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}\label{sec:RelatedWorks}

Our work belongs to the active research field of visual attention modeling,
for which a comprehensive discussion is beyond the scope of this paper.
We refer readers to recent survey papers for a detailed discussion of 65 models \cite{borji2012state},
as well as quantitative analysis of different methods in the two major research directions:
fixation prediction \cite{borji2012quantitative,Judd12} and salient object detection \cite{borjisalient}.


We focus on relevant literature targeting pre-attentive bottom-up saliency region detection,
which are biologically motivated, or purely computational, or involve both aspects.
%
Such methods utilize low-level processing to determine the contrast of image regions to their surroundings,
and use feature attributes such as intensity, color, and edges~\cite{09cvpr/Achanta_FTSaliency}.
%
We broadly classify the algorithms into local and global schemes.
Note that the classification is not strict as some of the research efforts can be listed under both categories.


Local contrast based methods investigate the rarity of image regions with respect to (small) local neighborhoods.
%
Based on the highly influential biologically inspired \emph{early representation} model introduced
by Koch and Ullman~\cite{85HN/KochVisualAttention}, Itti et al.~\cite{98pami/Itti} define image
saliency using central-surrounded differences across multi-scale image features.
%
Ma and Zhang~\cite{03ACMMM/Ma_Contrast-based} propose an alternate local contrast
analysis for generating saliency maps, which is then extended using a fuzzy growth model.
%
Harel et al.~\cite{conf/nips/HarelKP06} propose a bottom-up visual saliency model to normalize
the feature maps of Itti et al. to highlight
conspicuous parts and permit combination with other importance maps.
The model is simple, biologically plausible, and easy to parallelize.
%
Liu et al.~\cite{10pami/Liu_Learning} find multi-scale contrast by linearly combining contrast
in a Gaussian image pyramid.
%
More recently, Goferman et al.~\cite{10cvpr/goferman_context} simultaneously model local
low-level clues, global considerations, visual organization rules, and high-level
features to highlight salient objects along with their contexts.
%
Such methods using local contrast tend to produce higher saliency values near edges instead of uniformly
highlighting salient objects (see \figref{fig:cmp1vAll}).
Note that Reinagel et al.~\cite{99reinagel_natural} observe
that humans tend to focus attention in image regions with high spatial contrast and local variance in pixel correlation.


Global contrast based methods evaluate saliency of an image region using its contrast
with respect to the entire image.
%
Zhai and Shah~\cite{06acmmm/ZhaiS_spatiotemporal} define pixel-level saliency based on a pixel's contrast to all other pixels.
However, for efficiency they use only luminance information, thus ignoring distinctiveness clues in other channels.
%
Achanta et al.~\cite{09cvpr/Achanta_FTSaliency} propose a frequency tuned method that
directly defines pixel saliency using a pixel's color difference from the average image
color. The elegant approach, however, only considers first order average color, which
can be insufficient to analyze complex variations common in natural images.
%
In Figures~\ref{fig:VisualComparison} and~\ref{fig:plots}, we show qualitative and
quantitative weaknesses of such approaches. Furthermore, these methods ignore spatial
relationships across image parts, which can be critical for reliable and coherent
saliency detection (see \secref{sec:Experiment}).


Saliency maps are widely employed for unsupervised object segmentation:
%
Ma and Zhang~\cite{03ACMMM/Ma_Contrast-based} find rectangular salient regions
by fuzzy region growing on their saliency maps.
%
Ko and Nam~\cite{06josa/KoN_InterestSegmentation} select salient regions using a support
vector machine trained on image segment features, and then cluster these
regions to extract salient objects.
%
Han et al.~\cite{06TCSVT/han_unsupervised} model color, texture, and edge features in a Markov
random field framework to grow salient object regions from seed values in the saliency maps.
%
More recently, Achanta et al.~\cite{09cvpr/Achanta_FTSaliency} average saliency values within
image segments produced by mean-shift segmentation,
and then find salient objects via adaptive thresholding.
% by identifying image segments that have average saliency above
% a threshold that is set to be twice the mean saliency value of the entire image.
%
We propose a different approach that extends GrabCut \cite{04tog/rother_grabcut} method
and automatically initialize it using our saliency detection methods.
%
Experiments  on our $10,000$ images dataset (see \secref{sec:MSRA10K}) demonstrate the significant advantages
of our method compared to other state-of-the-art methods. %~\cite{11bmvc/Jiang_Saliency}.


Subsequent to our preliminary results~\cite{11cvpr/Cheng_Saliency},
Jiang et al.~\cite{11bmvc/Jiang_Saliency} propose a comparable method also
making use of region level contrast to model image saliency.
In the segmentation step, their method also expands and shrinks the
initial trimap and iteratively applies graphcut and histogram appearance model.
Since GrabCut is an iterative process of using graphcut and GMM appearance mode,
the two segmentation methods share a strong similarity.
Compared to the CB method \cite{11bmvc/Jiang_Saliency},
experimental results show that our RC salient object region detection and segmentation
is more accurate (\figref{fig:plots}(a)(c)),
$20\times$ faster (\figref{tab:TimeEfficency}),
and more robust to center-bias (\figref{fig:plots}(b)).
%There are, however, two main differences between the two approaches:
%(i)~Unlike our method, Jiang et al. use various segmentation parameters to get difference sized regions,
%and later combine the respective saliency maps.
%(ii)~Further, Jiang et al. make use of shape priors to refine saliency segmentation results.
%Since the range of immediate neighbor is closely associated with the segmentation,
%combining saliency maps obtained by difference segmentations can improve the
%saliency estimates.
%Not surprisingly, using spatial priors also improves our segmentation results.
%However, in absence of explicit information, we prefer not to use such a strong
%prior that can potentially produce biased results,
%e.g., in automated imaging systems like surveillance video.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Histogram Based Contrast}\label{sec:HC}

Our biological vision system is highly sensitive to contrast in visual signal. Based on this observation,
we propose a histogram-based contrast~(HC) method to define saliency values
for image pixels using color statistics of the input image.
%
Specifically, the saliency of a pixel is defined using its color contrast to all other pixels in the image,
i.e., the saliency value of a pixel $I_k$ in image $I$ is, % defined as,
\begin{equation}\label{equ:PixelColorContrast}
    S(I_k) = \sum\nolimits_{\forall I_i \in I} D(I_k, I_i),
\end{equation}
where $D(I_k, I_i)$ is the color distance metric between pixels $I_k$ and $I_i$ in the
\Lab space for perceptual accuracy. % Lab color space is more perceptual accurate than others.
%
%
\equref{equ:PixelColorContrast} can be expanded by pixel order as,
%to have the following form,
\begin{equation}\label{equ:PixelCCPixelOrder}
    S(I_k) = D(I_k, I_1) + D(I_k, I_2) + \cdots + D(I_k, I_N),
\end{equation}where $N$ is the number of pixels in image $I$.
%
It is easy to see that pixels with the same color have the same saliency
under this definition, since the measure is oblivious to spatial relations. Thus,
rearranging \equref{equ:PixelCCPixelOrder} such that the terms with the same color value
$c_j$ are grouped together, we get saliency value for each color as,
\begin{equation}\label{equ:PixelCCColorOrder}
    S(I_k) = S(c_l) = \sum\nolimits_{j=1}^{n}{f_j D(c_l, c_j)},
\end{equation}
where $c_l$ is the color value of pixel $I_k$, $n$ is the number of distinct pixel colors, and
$f_j$ is the probability of pixel color $c_j$ in image $I$.
%
%Note that in order to prevent salient region color statistics from being corrupted by
%similar colors from other regions, one can develop a similar scheme using varying window
%masks. However, given the strict efficiency requirement, we take the simple global
%approach.


\begin{figure}
    \centering
    \includegraphics[width=\columnwidth]{histogram.pdf} \\
    \TSpace
    \caption{Given an input image~(left), we compute its color histogram~(middle).
      Corresponding histogram bin colors are shown in the lower bar.
      The quantized image~(right) uses only  $43$ histogram bin colors
      and still retains sufficient visual quality for saliency detection.
    }\label{fig:colorFre}
    \BSpace
\end{figure}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Histogram based speed up}

Naively evaluating the saliency value for each image
pixel using \equref{equ:PixelColorContrast} takes $O(N^2)$ time, which is
computationally too expensive even for medium sized images.
%
The equivalent representation in \equref{equ:PixelCCColorOrder}, however, takes $O(N) + O(n^2)$ time,
implying that computational efficiency can be improved to $O(N)$ if $O(n^2) \leq O(N)$.
%
Thus, the key to speed up is to reduce the number of pixel colors in the image.
However, the true-color space contains $256^3$ possible colors,
which is typically larger than the number of image pixels.


Zhai and Shah~\cite{06acmmm/ZhaiS_spatiotemporal} reduce the number of colors, $n$,
by only using luminance. In this way, $n^2=256^2$ (typically $256^2 \ll N$).
The method, however, ignores distinctiveness of color information.
%
In this work, we use the full color space instead of luminance only.
To reduce the number of colors needed to consider,
we first quantize each color channel to have 12 different values,
which reduces the number of colors to $12^3=1728$.
%
Considering that color in a natural image typically covers only a small portion of the full color space,
we further reduce the number of colors by ignoring less frequently occurring colors.
%
By choosing more frequently occurring colors and ensuring these colors cover the colors of more
than $95\%$ of the image pixels, we typically are left with around $n=85$ colors (see
\secref{sec:Experiment} for experimental details).
%
The colors of the remaining pixels, which comprise fewer than $5\%$ of the image pixels,
are replaced by the closest colors in the histogram. A typical example of such
quantization is shown in \figref{fig:colorFre}.
%
Note that due to efficiency considerations we select the simple histogram based quantization
instead of optimizing for an image specific color palette.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Color space smoothing}\label{sec:smoothing}

Although we can efficiently compute color contrast by building a compact color histogram
using color quantization and choosing more frequent colors,
the quantization itself may introduce artifacts.
%
Some similar colors may be quantized to different values.
In order to reduce noisy saliency results caused by such randomness,
we use a smoothing procedure to refine the saliency value for each color.
%
We replace the saliency value of each color by the weighted average of the saliency
values of similar colors.% (measured by \Lab distance).
%
This is actually a smoothing process in the color feature space.
We choose
$m=n/4$ nearest colors to refine the saliency value of color $c$ by,
\begin{equation}\label{equ:smoothing}
    S'(c) = \frac{1}{(m-1)T} \sum\nolimits_{i=1}^{m} (T-D(c, c_i))S(c_i)
\end{equation}
where $T=\sum_{i=1}^{m} D(c, c_i)$ is the sum of distances between color $c$ and its
$m$ nearest neighbors $c_i$, and the normalization factor comes from
%\begin{equation*}\label{equ:smoothingN}
    $\sum_{i=1}^{m} (T-D(c, c_i))=(m-1)T.$
%\end{equation*}
%
Note that we use a linearly-varying smoothing weight $(T-D(c, c_i))$ to assign
larger weights to colors closer to $c$ in the color feature space.
%
In our experiments, we found that such linearly-varying weights are better than Gaussian
weights, which fall off too sharply.
%
\figref{fig:HCSmoothing} shows the typical effect of color space smoothing with the
corresponding histograms sorted by decreasing saliency values.
%
Note that similar histogram bins are closer to each other after such  smoothing,
indicating that similar colors is more likely to be
%have higher likelihood of being
assigned similar saliency values,
thus reducing quantization artifacts (see \figref{fig:plots}).



\begin{figure}
    \centering
    \includegraphics[width=\columnwidth]{histogram_saliency.pdf} \\
    \TSpace
    \caption{Saliency of each color %(normalized to $[0,1]$)
        before~(left) and after~(right) color space smoothing.
        Corresponding saliency maps are shown in the respective insets.
    }\label{fig:HCSmoothing}
    \BSpace
\end{figure}



\subsection{Implementation details}

To quantize the color space into $12^3$ different colors, we uniformly divide each color
channel into 12 levels.
%
While the quantization of colors is performed in the RGB color space, we measure color
differences in the \Lab color space given its perceptual accuracy.
%
We do not, however,  perform quantization directly in the \Lab color space since not all colors in
the range $L^*\in[0, 100]$, and $a^*, b^*\in[-127,127]$ necessarily correspond to real colors.
%
Experimentally we observed worse quantization artifacts using direct \Lab color space
quantization.
%
Best results were obtained by quantization in the RGB space while measuring distance in the \Lab
color space, as opposed to performing both quantization and distance calculation in a
single color space, either RGB or \Lab.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Region Based Contrast}


\begin{figure*}[t]
    \centering
    \begin{overpic}[width=\linewidth]{region_contrast.pdf}
        \put(6,0){(a)}
        \put(20,0){(b)}
        \put(35,0){(c)}
        \put(49,0){(d)}
        \put(64,0){(e)}
        \put(78,0){(f)}
        \put(91,0){(g)}
    \end{overpic} \\
    \TSpace
    \caption{{Region based contrast computation:
        (a)~input image, (b)~image regions generated by Felzenszwalb and
        Huttenlocher's segmentation method~\cite{04ijcv/felzenszwalb_efficient},
        (c)~region contrast without distance weighting and spatial prior (\equref{equ:regContrastSaliency}),
        (d)~region contrast with distance weighting,
        (e)~region contrast further considering spatial prior (\equref{equ:regContrastSpatial}),
        (f)~region contrast after improvement by border region estimation and color space smoothing,
        (g)~using our SaliencyCut (\secref{sec:salCut}),
        we get a high quality cut that is comparable to human labeled ground truth.
    }}\label{fig:regContrast}
    \BSpace
\end{figure*}

Humans pay more attention to image regions with high contrast to their
surroundings~\cite{03neuroscience/luminanceContrast}.
%
Besides contrast, spatial relationships are important in human attention.
%
High contrast to ones surrounding regions is usually stronger evidence for
saliency of a region than comparable contrast to far-away regions.
%
Since directly introducing spatial relationships when computing pixel-level contrast is
computationally expensive, we introduce a   contrast analysis method, \emph{region contrast} (RC),
so as to integrate spatial relationships into region-level contrast computation.
%
In RC, we first segment the input image into regions, then compute color contrast at the region level,
and finally define saliency for each region as the weighted sum of the region's contrasts
to all other regions in the image.
%
The weights are set according to the spatial distances with farther regions being assigned smaller
weights.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\subsection{Region contrast by histogram comparison}


We first segment the input image into regions using a graph-based image segmentation
method \cite{04ijcv/felzenszwalb_efficient}.
%
Then we build the color histogram for each region as in \secref{sec:HC}.
%
For a region $r_k$, we compute its saliency value by measuring its color contrast to all other
regions in the image,
\begin{equation}\label{equ:regContrastSaliency}
    S(r_k) = \sum\nolimits_{r_k \neq r_i} w(r_i)  D_r(r_k, r_i),
\end{equation}
where $w(r_i)$ is the weight of region $r_i$ and $D_r(\cdot, \cdot)$ is the color distance metric
between the two regions.
%
We weight the distances by the number of pixels in $r_i$ as $w(r_i)$ to emphasize color contrast to bigger regions.
%
The color distance between two regions $r_1$ and $r_2$ is,
% defined as,
\begin{equation}\label{equ:regContrast}
    D_r(r_1, r_2) = \sum\nolimits_{i=1}^{n_1} \sum\nolimits_{j=1}^{n_2} f(c_{1,i}) f(c_{2,j}) D(c_{1,i}, c_{2,j})
\end{equation}
where $f(c_{k,i})$ is the probability of the $i$-th color $c_{k,i}$ among all $n_k$ colors in the
$k$-th
 region $r_k$, $k=\{1,2\}$.
%
%Similarly we define $c_{2,j}$, $n_2$ and $f(c_{2,j})$ for region $r_2$.
%
Note that we use the probability of a color in the probability density function
(i.e., normalized color histogram) of the region as the weight for this color
to further emphasize the color differences between dominant colors.


Storing and calculating the regular matrix format histogram for each region is inefficient since
each region typically contains a small number of colors in the color histogram of the whole image.
%
Instead, we use a sparse histogram representation for efficient computation.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Spatially weighted region contrast}

We further incorporate spatial information by
introducing a spatial weighting term in \equref{equ:regContrastSaliency}
to increase the effects of closer regions and decrease the effects of farther regions.
%
Specifically, for any region $r_k$, the spatially weighted region contrast based saliency is:
%defined as:
\begin{equation}\label{equ:regContrastSpatial}
    S(r_k) = w_s(r_k) \sum\nolimits_{r_k \neq r_i} e^{  \frac{D_s(r_k, r_i)}{-\sigma_s^2} } w(r_i)  D_r(r_k, r_i)
\end{equation}
where $D_s(r_k, r_i)$ is the spatial distance between regions $r_k$ and $r_i$,
$\sigma_s$  controls the strength of spatial distance weighting,
$w(r_i)$ is the weight of region $r_i$ defined by the number of pixels in $r_i$,
and $w_s(r_k)$ is a spatial prior weighting term similar to center bias (\CB).
%spatial prior weighting term which prefers inner regions rather
%than border regions similar to \CB,}
%and $r_i$, and .
{We use $w_s(r_k) = \exp(-9 d_k^2)$, {where} $d_k$ is the average distance between
pixels in region $r_k$ and the center of the image, with pixel coordinates normalized to $[0, 1]$.  Thus, $w_s(r_k)$ gives a high value if region $r_k$ is close to the center of the image and it gives a low value if the region is a border region away from the center. }
%
For $\sigma_s$, larger values of $\sigma_s$ reduce the effect of spatial weighting
so that contrast to farther regions would contribute more to the saliency of the current region.
%
The spatial distance between two regions is defined as the Euclidean distance between their
centroids. In our implementation, we use $\sigma_s^2 = 0.4$ with pixel
coordinates normalized to  $[0, 1]$.

\subsection{Further improvement of RC saliency maps}\label{sec:ImproveRC}

\begin{figure*} [t!]
    \begin{overpic}[width=\linewidth]{saliency_cut.pdf}
        \put(7,1){(a)}
        \put(24,1){(b)}
        \put(41,1){(c)}
        \put(57,1){(d)}
        \put(74,1){(e)}
        \put(91,1){(f)}
    \end{overpic} \\
    \vspace{-.3in}
    \caption{{Demonstration of SaliencyCut: (a)~original image,
        (b)~initial segmentation by fixed thresholding the saliency map,
        (c)~trimap after first iteration, (d)~trimap after second iteration,
        (e)~final segmentation, and (f)~manually labeled ground truth.
        In the segmented images~(e), blue is foreground, gray is background, while in
        the trimaps~(b--d), foreground is red, background is green, and unknown regions are left unchanged.
    }}\label{fig:AttCutSteps}
    \BSpace
\end{figure*}

We further refine our RC saliency maps in two steps.
First, we use the spatial prior to explicitly estimate the non-salient (background) region.
Second, we apply the color space smoothing as described in \secref{sec:smoothing}.


We observe that regions with long borders overlapping with image borders are
typically non-salient background regions,
which we call border regions. We incorporate them as another spatial prior
($w_s(\cdot)$ in \equref{equ:regContrastSpatial})
to detect non-salient regions. In our implementation, we normalize the number of pixels located
in the 15 pixel-wide image-border area by the region size,
and consider regions with this value higher than a threshold to be border regions.
In practice, this hard constraint improves both the saliency maps as
well as the convergence speed of SaliencyCut (\secref{sec:salCut})
by improving the initial condition.
Our border region estimation aims at high precision, rather than high recall.
A strict fixed threshold,
which on average corresponds to $2\%$ miss alarm rate in our dataset,
is chosen to detect border regions.


In order to uniformly highlight the entire saliency region of the image,
we get the average saliency of each color in the color histogram
and adopt the color space smoothing (\secref{sec:smoothing}) to improve
our RC saliency map.
After smoothing, some border region pixels may get non-zero saliency values.
We reset the saliency of border region to zero and re-estimate the saliency
of each region as the average saliency value of its corresponding pixels.
Since initial RC maps are typically more uniformly highlighted compared to
HC saliency maps without color space smoothing,
we typically choose smaller number of nearest colors ($m=n/10$ in this part).
\figref{fig:regContrast}(f) demonstrates such an example.
The jumping man region is more uniformly highlighted compared to \figref{fig:regContrast}(e).




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{SaliencyCut: Automatic Salient Region Extraction}
\label{sec:salCut}


In a highly influential work, GrabCut~\cite{04tog/rother_grabcut} made critical
changes to the graphcut formulation to allow processing of noisy initialization.
This enabled users to roughly annotate (e.g., using a rectangle) a region of interest,
and then use GrabCut to extract a precise image mask.
Using our estimated saliency masks,
we remove even the need for user annotated rectangular regions.
In this section, we introduce {\em SaliencyCut}, which uses the computed saliency map
to assist in automatic salient object segmentation.
This immediately enables automatic analysis of large internet image repositories.
%
Specifically, we make two enhancements to GrabCut~\cite{04tog/rother_grabcut}:
``iterative refine" and ``adaptive fitting",
which together handle considerably more noisy initializations.
%
Thanks to the robustness of the new approach,
we are able to automatically initialize the segmentation
according to the detected saliency map.



\begin{figure*}
    \centering
    \small
    \renewcommand{\tabcolsep}{2.5mm}
    \begin{tabular}{l|c|c|c|c|c|c|c|c|c} \hline\hline
      Method    &  \IT   & \AIM   &  \IM   &  \MSS  & \SEG   & \SeR   & \SUN   &  \SWD   &  \CB          \\ \hline
      Time (s)  & 0.246  & 4.288  & 0.991  & 0.106  & 4.921  & 1.019  & 1.116  & 0.100   & 5.568         \\ \hline
      Code Type & Matlab & Matlab & Matlab &   C++  & Matlab & Matlab & Matlab &  Matlab & Matlab \& C  \\ \hline\hline
      Method    &   \GB  &  \SR   &  \FT   &  \AC   &  \CA   & \LC    &   HC   &  RC    &               \\ \hline
      Time (s)  & 1.614  & 0.064  & 0.102  & 0.109  &  53.1  & 0.018  & 0.019  & 0.254  &               \\ \hline
      Code Type & Matlab & Matlab &  C++   & Matlab & Matlab &  C++   &  C++   &  C++   &               \\ \hline\hline
    \end{tabular}
    \caption{Average time taken to compute a saliency map for images in the
        MSRA10K database (most have resolution $400\times 300$).
        {We use parallel computing environment for all Matlab functions for efficient
        computation.}
    } \label{tab:TimeEfficency}
\end{figure*}


\subsection{Algorithm initialization}
\label{sec:initialization}


Instead of manually selecting a rectangular region to initialize the process,
as in classical GrabCut,
we automatically initialize using a segmentation obtained by
binarizing the saliency map using a fixed threshold \cmm{$T_b$}.
Similar to GrabCut, we use incomplete trimap for the initialization.
\cmm{For image pixels with saliency value bigger than $T_b$,
the largest connected region is
considered as initial candidate region of the most dominate salient object.
This candidate region is labeled as unknown part of the trimap,
while other regions are labeled as background.}
Note that we do not initialize any hard foreground labeling.
These unknown regions are initially used to train foreground color models
thus helps the algorithm to identify the foreground pixels.


Since the initial background regions are retained while other regions
may be changed during the GrabCut optimization,
we give preference to confident background labels in the trimaps.
Thus we initialize the GrabCut algorithm using threshold given high
recall of potential foreground region and let the iterative
optimization process to increase its precision.
In our experiments, the threshold is chosen empirically to be the threshold
that gives $95\%$ recall rate in our fixed thresholding experiments (see \secref{sec:fixThr}).
When initialized using RC saliency maps,
\cmm{we use $T_b=70$} with saliency values normalized to $[0, 255]$.




\subsection{Segmentation by iterative fitting}


Once initialized, we iteratively run GrabCut \cite{04tog/rother_grabcut}
to improve the SaliencyCut result~(maximum of $4$ iterations in our experiments).
After each iteration, we use dilation and erosion operations on the current segmentation
result to get a new trimap for the next GrabCut iteration.
As shown in \figref{fig:AttCutSteps}(c, d), the region outside the dilated region is set to
background, the region inside the eroded region is set to foreground, and the remaining
areas are set to unknown in the trimap.
GrabCut, which by itself is an iterative process using Gaussian mixture models and graph-cut \cite{boykov2001fast},
helps to refine salient object regions at each step.


Different from one-pass GrabCut or the even simpler graph cut based saliency segmentation \cite{mehrani2010saliency},
the new scheme in SaliencyCut \emph{iteratively refines} the initial salient regions.
Such an iterative design is important to handle noisy initializations supplied by
the saliency detection algorithm rather than human annotations.
In case of incorrect initialization as shown in flower example in \figref{fig:AttCutSteps}~(b),
the initial background region incorrectly contains foreground object(s).
Although we can still get a segmentation result containing many parts of the flower
using GrabCut, the remaining flowers in the initial background region would never be correctly
extracted using GrabCut since the background gets a hard labeling.
%Such problematic initialization, which is unavoidable for automatic method,
%we need to make the algorithm to be robust to poor initialization.
One may consider relaxing the hard constrain of GrabCut to solve this problem.
However, experimental results show this would make the method not stable,
often producing results containing all foreground or all background.

We iteratively refine the initial segmentation and adaptively change
the initial condition to fit with newly segmented salient region.
The \emph{adaptive fitting} is based on an important observation:
regions closer to an initial salient object
region are more likely to be part of that salient object than far-away regions.
Thus, our new initialization enables GrabCut to include nearby salient regions,
and exclude non-salient regions according to color feature dissimilarity.
After each GrabCut iteration,  SaliencyCut
incorporates the constraints given by the newly obtained trimap,
and train a better appearance model according to previous results.


\figref{fig:AttCutSteps} shows three examples.
In the flower example, SaliencyCut successfully expanded the initial salient regions
(obtained directly from the saliency map) and converged to an accurate segmentation result.
In the excavator and teapot examples,
unwanted regions are correctly excluded during GrabCut iterations.
The intermediate steps show how  SaliencyCut  successfully
extracted the object regions of interest in these challenging examples.
%
A comprehensive quantitative evaluation of different saliency segmentation methods is presented in \secref{sec:EvalSalCut}.


\begin{figure}[b]
  \centering
  \includegraphics[width=\linewidth]{DataSet.pdf}\\
  \TSpace
  \caption{Ground truth examples: (first row) original images with
    ground truth rectangles from \cite{10pami/Liu_Learning},
    (second row) our ground truth, which have more precisely marked important
    regions at pixel level accuracy.
  }\label{fig:gt}
\end{figure}

\begin{figure*}[t!]
   \begin{overpic}[width=\textwidth]{comparison.pdf}
   \put(2,0){(a)~original}
   \put(18,0){(b)~LC}
   \put(33,0){(c)~CA}
   \put(47,0){(d)~FT}
   \put(58.5,0){(e)~\HC}
   \put(73,0){(f)~\RC}
   \put(89,0){(g)~RCC}
   \end{overpic}\\
   \vspace{-.23in}
   \caption{Visual comparison of saliency maps. (a)~original images, saliency maps produced using
       (b)~Zhai and Shah~\cite{06acmmm/ZhaiS_spatiotemporal},
       (c)~Goferman et al.~\cite{10cvpr/goferman_context},
       (d) Achanta et al.~\cite{09cvpr/Achanta_FTSaliency},
       (e) our HC and (f)~RC methods, and (g) SaliencyCut.
       Our methods generate uniformly highlighted salient regions
       (see our project webpage for all results on the full benchmark dataset).
   }\label{fig:VisualComparison}
   \BSpace
\end{figure*}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental Comparisons}\label{sec:Experiment}



In this work, we extensively evaluated our saliency detection method
on three different types of benchmark datasets,
and compared it against 15 alternate methods
--- \SR, \IT, \IM, \SUN, \AC, \SeR, \AIM,
\GB, \LC, \CA, \FT, \SWD, \SEG, \MSS, \LP~ and \CB, respectively.
Following~\cite{09cvpr/Achanta_FTSaliency}, we selected these methods according to:
number of citations (IT, SR, SUN, AIM ~and FT),
recency (SeR, MSS, SEG, IM, CA ~and SWD),
variety (IT is biologically-motivated, LC is purely computational, GB and LP are hybrid,
SR works in the frequency domain,
AC and FT output full resolution saliency maps),
and being related to our approach (LC and CB).


\tabref{tab:TimeEfficency} compares the average time taken by each method
on a Dual Core 2.6 GHz machine with 2GB RAM.
%
Our algorithms, HC and RC, are implemented in C++.
%
For the other methods namely IT, AIM, IM, MSS, SEG, SeR, SUN, GB, SR, AC, CA,
FT and CB, we used the authors' implementations,
while for LC, we implemented the algorithm in C++ since we failed to obtain the authors'
implementation.
%
For typical natural images, our HC method runs in $O(N)$, which is sufficient for real-time applications.
In contrast, our RC variant is slower as it requires image
segmentation~\cite{04ijcv/felzenszwalb_efficient}, but produces superior quality saliency maps.


The true effectiveness of a saliency detection method depends
on the applications \cite{09cvpr/Achanta_FTSaliency}.
We evaluated our method on \cmm{three core computer vision and
graphics applications, namely
salient region segmentation by fixed thresholding, % content aware image resizing, non-photorealist rendering, adaptive image compression, image mosaic
object of interest image segmentation, % Object detection, object recognition, auto-cropping
and sketch based image retrieval (SBIR).}  % Image composition, recoloring, view estimation, image collage, object manipulation
%12 other applications which naturally be benefited by the advantages
%demonstrated on these core applications are also discussed.


%In order to comprehensively evaluate the accuracy of our methods for salient object
%segmentation, we performed two experiments using different objective comparison measures.
%%In the first experiment
%\cmm{Firstly}, to segment salient objects and calculate precision and recall
%curves, we binarized the saliency map using every possible fixed threshold
%(similar to \cite{09cvpr/Achanta_FTSaliency}).
%%In the second experiment
%\cmm{Secondly}, we segment salient objects by our SaliencyCut approach (\secref{sec:salCut}).
%%
%\cmm{additional experiments (SBIR) are not described there}


\subsection{Benchmark datasets for saliency detection}




\subsubsection{Images with unambiguous salient objects} \label{sec:MSRA10K}

Similar to existing salient object region detection methods
\cite{09cvpr/Achanta_FTSaliency,10icip/Achanta_saliency,10pami/Liu_Learning,11bmvc/Jiang_Saliency},
we first evaluate our methods on images with unambiguous salient object.
The largest dataset of this kind is provided by Liu et al. \cite{10pami/Liu_Learning}.
This dataset contains 20,000+ images (mostly at $400\times 300$ resolution),
with bounding box labeling by 3-9 users.
\cmm{These images are selected from an initial set of 130,099 images,
such that each image contains a clear, unambiguous object of interest.}
Since objects can still be recognized at low resolution,
the dataset has limited scale and location variations of salient objects,
i.e., implicitly the images have scale and location priors (Flickr like).

%We derived our single salient object images database from the publicly available
%database provided by Liu et al. \cite{10pami/Liu_Learning},
%which contains 20,000+ images (mostly at $400\times 300$ resolution).
%Since objects can still be recognized at low resolution, the dataset has limited
%scale and location variations of salient objects,
%i.e., implicitly the images have scale and location priors.
%Each image in this database contains an unambiguous salient object, which was consistently
%annotated with bounding boxes by 3-9 users.

Although an invaluable recourse to evaluate saliency detection algorithms,
the database with the marked bounding boxes, however,
is often too coarse for fine grained evaluation as observed by
Wang and Li~\cite{08icassp/Wang}, and Achanta et al.~\cite{09cvpr/Achanta_FTSaliency}.
In order to do more extensive and accurate evaluation,
we randomly selected 10,000 images with consistent bounding box labeling in MSRA database
provided by Liu et al.~\cite{10pami/Liu_Learning} and the consistent measure is the same
as choosing image dataset B in their paper.
As shown in \figref{fig:gt}, we accurately marked pixels in salient object regions.
We call this dataset MSRA10K because it contains 10,000 images with {\em pixel-level}
saliency labeling (publicly available on our project page).
Our dataset is 10 times bigger than what was previously the largest public available database
of its kind~\cite{09cvpr/Achanta_FTSaliency}. % with pixel-level salient region marking.
In our experiments, we find that saliency detection methods using pixel level contrast
(FT, HC, LC, MSS) do not scale well on this lager benchmark (see \figref{fig:plots}(a)),
suggesting the importance of region level analysis.
%When choosing the subset of images from \cite{10pami/Liu_Learning}
%for pixel accuracy labeling,
%Achanta et al.~\cite{09cvpr/Achanta_FTSaliency} choose those images
%which can be easily segmented.
%This might be the reason why their dataset is biased towards saliency detection
%methods which simply rely on per-pixel color difference
%(see the significant performance drop of FT, MSS, HC when using
%our MSRA10K dataset in \figref{fig:plots}(a)).




\subsubsection{Randomly-selected internet images}\label{sec:THUR15K}


While state-of-the-art methods consistently produce excellent results when
evaluated using the traditional benchmark dataset \cite{09cvpr/Achanta_FTSaliency}
(see \figref{fig:plots}(c)),
ordinary users often report less satisfactory experiences when using their own images.
This encourage us to think about two questions:
`How would these methods deal with random internet images?' and
`When can we trust the results of these methods?'
To better explore these issues, we evaluated salient object segmentation methods
on a dataset with randomly-selected internet images \cite{ChengGroupSaliency}.
This benchmark dataset, namely THUR15K \cite{ChengGroupSaliency},
contains about 3000 images downloaded from
\href{http://www.flickr.com/}{Flickr} for each of the 5 keywords:
``butterfly,"  ``coffee mug,"  ``dog jump,"  ``giraffe,"  and ``plane."
Salient regions in THUR15K images are  marked at pixel accuracy.
Note that not every image in the THUR15K dataset contains a salient region label,
as some images do not have any salient object region.
Besides saliency detection,
this dataset can also be used to evaluate the performance of sketch based image retrieval (SBIR).


%We also introduce a labeled dataset of categorized images for evaluating sketch based image retrieval.
%Using \href{http://www.flickr.com/}{Flickr},
%we downloaded about 3000 images for each of the 5 keywords:
%``butterfly", ``coffee mug", ``dog jump", ``giraffe", and ``plane",
%together comprising of about 15000 images.
%For each image, if there is a non-ambiguous object with correct content matching with
%the query keyword and most part of the object is visible, we mark such an object region.
%Similar to MSRA10K, the salient regions are marked at a pixel level.
%We only label salient object region for objects with almost fully visible
%since partially occluded objects are is less useful for shape matching.
%In contrast to MSRA10K, the THUR15K dataset does not contain a salient region
%labeled for every image in the dataset, i.e., some images do not have any salient region.
%This dataset is used to evaluate shape based image retrieval performance.




\subsubsection{Human fixation dataset}\label{sec:JuddDataSet}

While our algorithm targets salient object detection,
it is also interesting to test its performance on human fixation prediction
benchmarks.
We use the most widely adopted human fixation benchmark \cite{Judd_2009} for such evaluation.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\subsection{Segmentation by fixed thresholding} \label{sec:fixThr}

The simplest way to get a binary segmentation of salient objects is to
threshold the saliency map with a threshold $T_f \in [0, 255]$.
%
To reliably compare how well various saliency detection methods highlight salient regions in images,
we vary the threshold $T_f$ from $0$ to $255$.
\figref{fig:plots}(a) shows the resulting precision vs. recall curves.
%
%\cmm{\st{We also present objective comparison with other saliency extraction methods.}}
%
Typical qualitative comparison of saliency maps obtained by the various methods are presented in
\figref{fig:cmp1vAll} and \figref{fig:VisualComparison}.
%(All test results on the whole database are included as supplementary material.)


Unlike most other methods, both the CB method and our RC method use the center location prior of the
man-made photographs.
However, for a fair comparison, \figref{fig:plots}(b) shows comparisons while disabling such a location prior.
Specifically, RC1 shows the effect disabling the center location weighting (\equref{equ:regContrastSpatial}) of RC method,
while RC2 shows the effect of further disabling border region estimation (\secref{sec:ImproveRC}).
Other methods in \figref{fig:plots}(b) also improve when we use the same segmentation,
as used in RC,
to average saliency values within each segment and re-normalize to $[0, 255]$ by uniform scaling.
Note that many of these methods aim to predict human eye movements
rather than perform salient object segmentation, as is our focus.




\begin{figure}[b]
    \centering
    \includegraphics[height=.087\textwidth]{Fixation/Img}
    \includegraphics[height=.087\textwidth]{Fixation/Map}
    \includegraphics[height=.087\textwidth]{Fixation/RCNB}
    \includegraphics[height=.087\textwidth]{Fixation/RCWB}
    \\
    \TSpace
    \caption{From left to right, we show source image,
        ground truth eye fixation map by human observer,
        our RC result with the term encouraging similar appearance region receive
        similar saliency (\secref{sec:ImproveRC}) disabled,
        and result by our full RC method.
    }\label{fig:fixation}
\end{figure}


\begin{figure}[b!]
    \centering
    \small
    \BSpace
    \renewcommand{\arraystretch}{1.1}
    \renewcommand{\tabcolsep}{3.1mm}
    \begin{tabular}{l|c|c|c|c} \hline\hline
      Method   &  \MSS  &  \CA  &  \LP  &  Ours   \\ \hline
      ROC Area & 0.683  & 0.844 & 0.849 &  0.830  \\ \hline\hline
    \end{tabular}
    \caption{Comparison on human fixation dataset \cite{Judd_2009}.} \label{tab:Fixation}
\end{figure}

\begin{figure*}[t!]
    \centering
    \begin{tabular*}{\linewidth}{ccc}
        \begin{sideways} ~~~~~~~~~~~~~ (a) Original methods \end{sideways} &
        \includegraphics[width=0.45\linewidth]{plots/MapsFT} &
        \includegraphics[width=0.45\linewidth]{plots/MapsTHUS} \\
        \begin{sideways} ~~~~~~ (b) Variants of these methods \end{sideways} &
        \includegraphics[width=0.45\linewidth]{plots/MapsVarFT} &
        \includegraphics[width=0.45\linewidth]{plots/MapsVarTHUS}
        \\
        \begin{sideways} ~~~~ (c) Saliency segmentation methods \end{sideways} &
        \begin{overpic}[width=0.45\linewidth]{plots/CutFT} \PutOverDes \end{overpic} &
        \begin{overpic}[width=0.45\linewidth]{plots/CutTHUS}  \PutOverDes \end{overpic}
        \\
        & (i) Achanta et al. \cite{09cvpr/Achanta_FTSaliency} dataset ($1, 000$ images)  &  (ii) MSRA10K dataset ($10,000$ images)
    \end{tabular*}
    \caption{Statistical comparison results of (a)~different saliency detection methods,
        (b)~their variants, and  (c)~object of interest region segmentation methods,
        using largest public available dataset~(i)
        and (ii)~our MSRA10K dataset.
        %(Please refer to our project webpage for details.)
    }\label{fig:plots}
    \BSpace
\end{figure*}


The precision and recall curves clearly show that our RC method outperforms the other methods.
We observe a significant loss in precision \figref{fig:plots}(b) for the CB method
(which has best performance in the benchmark paper \cite{borjisalient})
indicating that the method heavily relies on location prior.
%
The extremities of the precision vs. recall curve are interesting: At maximum recall
where $T_f = 0$, all pixels are retained as positives, i.e., considered to be
foreground, so all the methods have the same precision and recall values; precision
$0.22$ and recall $1.0$ at this point indicate that, on average, there are $22\%$ image
pixels belonging to the ground truth salient regions.
%
At the other end, the minimum recall values of our RC method are higher than those of the other
methods, because the saliency maps computed by our RC method are smoother and contain more
pixels with the saliency value $255$.
Our HC method also has better precision and recall compared to methods with
similar computational efficiency (SR, FT, and LC).
After comparison of a large number of saliency detection models,
Borji et al. \cite{borjisalient} proposed a combined model and
show that integration of the few best models
(with the initial version of our method as one of them) outperforms all models.
\reviewer{We believe that the combined model of \cite{borjisalient} will further benefit from
performance improvement due to our method.}




As also observed in the survey papers \cite{borji2012state,borji2012quantitative,Judd12,borjisalient},
center-bias naturally exists in human captured photos.
Judd et al. \cite{Judd_2009} further found that a simple Gaussian blob
performs better than many saliency detection methods when evaluated in famous eye
fixation dataset.
We experimentally find that such simple Gaussian blob, represented by `Gau' in \figref{fig:plots}(a)(c),
also performs better than many existing models for saliency region detection task.
However, in the absence of explicit information, we prefer not to use such a strong prior
that can potentially produce biased results, e.g., in automated imaging systems.
When disabling the center bias term, our method still produces better
results than other alternatives \figref{fig:plots}(b).




In the context of fixation prediction, the \CA~ and \LP~ methods report the
best performance.
Although it avoids the heavy learning for combining multi-saliency models and object detectors,
the CA method still needs about 1 min to calculate a saliency map even for small images.
\tabref{tab:Fixation} and \tabref{tab:TimeEfficency} shows that our method,
although initially designed for saliency region detection,
has only slightly lower performance to state-of-the-art methods for predicting human
fixation points, while being 200+ times more efficient.
Readers can refer to \cite{Judd_2009,Judd12,10cvpr/goferman_context} for more comparisons.
Notice that the good performance of our RC method for predicting eye fixation points
shown in \tabref{tab:Fixation} is achieved by disabling the term encouraging
similar appearance region receive similar saliency value,
thus improves human fixation point prediction as demonstrated in \figref{fig:fixation}.
%to quickly remember an image for distinguishing
%whether the image has been presented before,
%a human observer has to fix on some important parts of the image objects.
%In this example, to notice the head of the horse to make sure it is not some other animal.
Although disabling the process explained in \secref{sec:ImproveRC}  improves
eye fixation prediction performance,
we argue that uniformly highlighting the entire object region is better in many applications,
including content aware image resizing \cite{09cgf/ZhangC},
non-photorealist rendering \cite{11cvpr/Cheng_Saliency},
adaptive image compression \cite{00CE/christopoulos_jpeg},
and image mosaic \cite{10cvpr/goferman_context}.
Thus, although their own method \cite{10cvpr/goferman_context} achieves best performance on
eye fixation dataset \cite{Judd_2009},
Margolin et al. \cite{margolin2012saliency} still choose to integrate our RC saliency
maps to achieve better effects for various of image manipulation applications.







\subsection{Object of interest image segmentation} \label{sec:EvalSalCut}


To objectively evaluate our new SaliencyCut method using our RC-map as initialization,
we compare our results with results obtained by
{other state-of-the-art methods for object of interest segmentation, i.e., \FT, \SEG,
GrabCut\GRAB ~(initialized using 5 pixel wide image boundary), and \CB~
(best parameters are selected for these methods).
%
Average precision, recall, and $F$-Measure are compared against the entire ground-truth
database~\cite{09cvpr/Achanta_FTSaliency}, with the $F$-Measure defined as:
\begin{equation}\label{equ:FMeasure}
    F_{\beta} = \frac{(1+\beta^2)Precision \times Recall}{\beta^2 \times Precision + Recall}.
\end{equation}
We use $\beta^2 = 0.3$ as in Achanta et al.~\cite{09cvpr/Achanta_FTSaliency} to weigh
precision more than recall.
%
As can be seen from the comparison (see \figref{fig:plots}(c)),
SaliencyCut using our RC saliency maps significantly outperforms other methods.
%
As discussed by Liu et al. \cite{10pami/Liu_Learning},
recall rate is not as important as precision for attention detection.
For example, a $100\%$ recall rate can be achieved by simply selecting the whole image.
Our approach reduced $57.2\%$, $50.9\%$, $46.5\%$, and $23.7\%$ overall error rates on F-measure,
compared with \FT, \SEG, GrabCut\GRAB, and \CB, respectively when evaluated using large accurate dataset (MSRA10K).
%As a subsequent work to our initial reporting \cite{11cvpr/Cheng_Saliency},
%the saliency segmentation in \CB ~has variations of most of our technical features.
Besides producing higher F-Measure and robustness to location prior,
our SaliencyCut %(demo software available on project webpage)
is about 60 times faster
(see \tabref{tab:TimeEfficencySEG}) compared to \CB.
%which is a subsequent work to our initial reporting \cite{11cvpr/Cheng_Saliency}.%}
\cmm{Although several new methods \cite{yan2013hierarchical,GrabCutOneCut} have
been developed since the initial version of this work \cite{11cvpr/Cheng_Saliency},
to the best of our knowledge, our salient object segmentation results
are still the best results reported on the most widely used benchmark \cite{09cvpr/Achanta_FTSaliency}.}


\begin{figure}[b]
  \centering
  \begin{overpic}[width=.99\linewidth]{plots/CutCmpR.pdf}
       \small
       \put(7,  1){Butterfly}
       \put(25, 1){CoffeeMug}
       \put(46, 1){DogJump}
       \put(66, 1){Giraffe}
       \put(86, 1){Plane}
  \end{overpic}\\
  \vspace{-.08in}
  \caption{Comparison of average $F_{\beta}$ for different saliency segmentation methods:
    FT~\cite{09cvpr/Achanta_FTSaliency},
    SEG~\cite{10eccv/rahtu_segmenting}, and ours,
    on THUR15K dataset \cite{ChengGroupSaliency}.
  } \label{fig:plotsCutR}
\end{figure}


\begin{figure}
    \centering
    \small
    \renewcommand{\arraystretch}{1.1}
    \renewcommand{\tabcolsep}{2.5mm}
    \begin{tabular}{l|c|c|c|c} \hline\hline
      Method  &  \FT   &      \SEG     &      \CB      &  Ours   \\ \hline
      Time (s) & 0.247  &     7.48      &     36.5      &  0.621 \\ \hline
      Code    & Matlab & Matlab \& C & Matlab \& C &  C++   \\ \hline\hline
    \end{tabular}
    \caption{{Comparison of average time taken for different saliency
        segmentation methods. \cmm{Segmentation results for MSRA10K
        dataset are available via our project page.}}}
    \label{tab:TimeEfficencySEG}
\end{figure}

\begin{figure*}
    \centering
    \includegraphics[width=.99\textwidth]{GiraffeH}
    \TSpace
    \caption{Sketch based image comparison: first row shows images downloaded from
        \href{http://www.flickr.com/}{Flickr} using keyword `giraffe',
        second row shows our retrieval results obtained by comparing user input
        sketch with SaliencyCut result using shape context measure \cite{02pami/BelongieMP};
        third row shows corresponding sketch based retrieval results using \SHoG.
        %(see our project page for more examples.)
    }
    \label{fig:SBIR_demo}
\end{figure*}

\begin{figure*}
    \begin{center}
    \small
    \renewcommand{\arraystretch}{1.1}
    \renewcommand{\tabcolsep}{3.56mm}
    \begin{tabular}{c||c|c|c|c|c|c|c|c|c|c|c|c} \hline\hline
      \Rows{TPR (\%)}         & \Cols{Butterfly} & \Cols{Coffee mug} & \Cols{Dog jump} & \Cols{Giraffe} & \Cols{Plane}   & \mcols{\textbf{Average}}      \\ \cline{2-13}
                              &    \colInf       &      \colInf      &      \colInf    &      \colInf   &       \colInf  & \textbf{T50}  & \textbf{T100} \\ \hline
      Flickr                  &     28   &  28   &     58   &   51   &     56   & 55   &    30  &  25   &    44  &   48  & \textbf{41.4} & \textbf{43.2} \\ \hline
      %Ours ($F_\beta > 0.7$) &     36   &  32   &     86   &   91   &     64   & 55   &    66  &  62   &    68  &   68  & \textbf{61.6} & \textbf{64.0} \\ \hline
      Ours                    &     58   &  52   &     88   &   93   &     86   & 90   &    72  &  66   &    88  &   90  & \textbf{78.2} & \textbf{78.4} \\ \hline
      \SHoG                   &     36   &  40   &     82   &   78   &     74   & 73   &    18  &  18   &    90  &   91  & \textbf{60.0} & \textbf{60.0} \\ \hline\hline
    \end{tabular}
    \end{center}
    \vspace{-.05in}
    \caption{True positive ratios (TPR) among top $50$ and $100$ retrieval results.
        Results for SHOG are supplied by original authors.
        An image is considered as true positive if it contains a target object specified by the keywords.
    }
    \label{tab:statistic}
    %\BSpace
\end{figure*}



Although producing quite promising results for simple images as evaluated in
\figref{fig:plots},
evaluation results for randomly-selected internet images \figref{fig:plotsCutR} shows that
there is still a need to develop more robust methods.
For both datasets, our SaliencyCut's performance is the best.
%We believe that such high performance in predicting the entire salient object region
%can benefit object recognition \cite{fraundorfer2003detecting},
%classification \cite{04cvpr/RutishauserWWKP}, and auto-cropping \cite{margolin2012saliency}.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Sketch based image retrieval}



Outline sketches are typically easier and faster for users to generate
than a complete color description of the desired image.
Sketch based image retrieval (SBIR) techniques become vital for users to
leverage the increasing volumes of available image database.
A large majority of potential users fail to precisely express fine
details in their drawings.
Thus most SBIR systems, which employ global descriptors,
are unsatisfactory as they are unreliable under affine variations.
To overcome such drawbacks, Eitz et al.~\cite{10cg/eitz_eval_dis,11tvcg/eitz_sbir}
use local descriptors to achieve state-of-the-art retrieval performance.
The success of their methods is mainly attributed to translation invariance
of local descriptors while using large local feature size (in the order
of $20-25\%$ of the image's diagonal) to still retain large scale characteristics.
However, for such large window sizes, there is simply not much space left for
translating the sketch, thus limiting the translation invariance.
SBIR still suffers from relatively low accuracy thus restricting its
commercial potential.




Matching object shapes with clean background, however, is a relatively mature field.
Even for the very challenging MPEG-7 dataset, state-of-the-art methods can
achieve $91.61\%$ retrieval rates \cite{10pami/bai_learning}.
Classical shape methods such as Shape Contexts (SC) \cite{02pami/BelongieMP} and Chamfer
Matching \cite{03cvpr_ShapeContextAndChamfer} are mostly successful when dealing with
limited background clutter.
Selecting clean object outlines without influence from irrelevant image edges
has great potential to improve current SBIR systems.
Based on the observation that good results cannot be achieved without selection of segments,
Bai et al. \cite{09cvpr/Bai} use a shape band model to coarsely select candidate
of edge segments while using Shape Context distance to decide the optimal matching.
However, the shape band model requires user sketch for further detection thus
does not allow preprocessing.
It needs a few minutes to process a single image making it unsuited for real-time
image retrieval applications.




Our SaliencyCut algorithm provides another possibility for automatically finding
the outlines of object of interest on large scale image datasets.
After such preprocessing, it becomes possible to make use of proven shape matching algorithms.
We simply rank the images by SC \cite{02pami/BelongieMP} distance between their
salient region outlines and user input sketches
and compare with a state-of-the-art SBIR method using \SHoG.

Experiments indicate that although our SaliencyCut method may
produce less optimal results for noisy internet images,
the shape matching method is very efficient in selecting those well segmented results.
A quantitative evaluation in our THUR15K dataset is shown in \tabref{tab:statistic}.
One can see that our retrieval method is more effective than SHoG in terms of selecting user-desired candidates.
Sample qualitative results are shown in \figref{fig:SBIR_demo}.
Compared with SHoG, our method gives results that are more relevant to user input sketches.
Moreover, our method produces the precise boundary of the desired object,
which makes it possible to reuse these segmented image components in many applications.
Note that the extracted salient region features are complementary to other features like color,
texture, and local edges.
% and can be integrated with existing features for content/sketch based image retrieval.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\section{Discussion}

We presented global contrast based saliency computation methods,
namely Histogram based Contrast~(HC)
and spatial information-enhanced Region based Contrast~(RC).
%
While the HC method is fast and generates results with fine details,
the RC method generates spatially coherent high quality saliency maps
at the cost of reduced computational efficiency.
%
\cmm{Based on the proposed saliency detection method,
we  introduced a novel unsupervised segmentation algorithm,
namely SaliencyCut, to automatically segment
the most salient object in an image,
without requiring expensive training data.
%
The proposed methods were evaluated on several large scale
publicly available benchmarks.
The experimental results show that our methods consistently outperform
other state-of-the-art methods in terms of precision and recall,
while still being simple, fast, and efficient.
%
For noisy internet images,
although our saliency detection and segmentation methods cannot guarantee
robust performance on individual images,
their efficiency and simplicity makes it possible to
automatically process a large number of images,
which can then be further filtered for reliability and accuracy.
}}




\cmm{\myPara{Limitations} Our methods aim at finding
the most salient object in an image.
It might produce sub-optimal results for images with multiple objects (e.g., \figref{fig:multiOjb}),
especially if the objects occlude each other (e.g., PASCAL VOC images \cite{everingham2010pascal}),
for which even specialized object detectors fail to
reliably generate good results for most object classes.
Rather than making hard decisions early on,
proposing some candidate object regions
\cite{alexe2012measuring,endres2013category,BingObj2014}
can be useful for those applications requiring high detection
rate, e.g., object detection in cluttered scenes.}



\newcommand{\addFigure}[1]{\includegraphics[height=0.053\textwidth]{SED2S/{#1}}}
\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{SED2S}
  \vspace{-.2in} \\
  \caption{\cmm{Salient object detection results
    for benchmark images \cite{AlpertGBB07} with multiple objects.}
  }\label{fig:multiOjb}
\end{figure}


\cmm{\myPara{Future work}
We further discuss possible applications and extensions
by highlighting a few of the many exciting works using the preliminary
version of our work:
%(see citations of \cite{11cvpr/Cheng_Saliency} for more details):
%http://scholar.google.com/citations?view_op=view_citation&citation_for_view=huWpVyEAAAAJ:MXK_kJrjxJIC
\begin{itemize}
  \item The ability to generate high quality saliency maps
    is essential for many applications including content-aware image manipulation,
    \cite{panozzo2012robust,mai2011rule},
    non-photorealist rendering \cite{rosin2013artistic,han2013fast},
    image scene analysis~\cite{borji2013stands,zhang2013discriminative,AttObj2014}
    adaptive compression \cite{deng2012content},
    forgery detection \cite{fu2012forgery,ChangeBlindness13}, etc.
  \item Unsupervised segmentation of the entire salient object,
    without extensive training data annotation,
    naturally benefits applications like
    auto-cropping \cite{margolin2012saliency},
    scene classification \cite{ji2013balance},
    semantic manipulation \cite{wang2013affective,cheng2013imagespirit,SemanticPaint},
    and data-driven image synthesis \cite{goldberg2012data,tong2013stereopasting}. % chen2012visual,zhang2012storywizard
  \item A tool to retrieve internet images and get precise
    object of interest regions is powerful to explore this big data for
    image composition \cite{11tog/HuangArcimboldo},
    semantic colorization \cite{11tog/ChiaColorization},
    information discovery \cite{11tvc/Liu,zhu2012unsupervised},
    image retrieval \cite{he2012mobile,tvcg/Chen,gao20123}, %li2013combining,
    etc.
  \item The proposed saliency measure has already been used to
    produce state-of-the-art results on cosegmentation benchmarks
    without using cosegmentation \cite{rubinstein2013unsupervised},
    or simultaneously analyze multiple images for better salient object extraction
    \cite{fu2013cluster,ChengGroupSaliency}.
\end{itemize}
}





\section*{Acknowledgments}

We would like to thank Prof. Z. Tu (for the suggestion
of region level analysis), Prof. C. Rother, Prof. A. Zisserman,
and the anonymous reviewers for their many useful feedbacks.
This research was supported by the 973 Program (2011CB302205),
NSFC (61120106007, 61133008), the 863 Program (2012AA011802),
Research program of Beijing Education council,
EPSRC, Leverhulme Trust, ERC (2012-AdG 321162-HELIOS),
and ERC Starting Grant SmartGeometry.


\ifCLASSOPTIONcaptionsoff
  \newpage
\fi

\bibliographystyle{IEEEtran}
\bibliography{Saliency}


%\vspace{-.4in}
%\begin{IEEEbiography}[{\includegraphics[width=1in]{./Author/cmm}}]
%{Ming-Ming Cheng} received his PhD degree from Tsinghua University in 2012.
%He is currently a research fellow in The University of Oxford,
%working with Prof. Philip Torr. His research interests includes computer graphics,
%computer vision, image processing, and image retrieval.
%He has received the Google PhD fellowship award, the IBM PhD fellowship award, and the “New
%PhD Researcher Award” from Chinese Ministry of Education.
%\end{IEEEbiography}
%\vspace{-.45in}

\vspace{-.4in}
\begin{IEEEbiography}[{\includegraphics[width=1in]{./Author/cmm}}]
{Ming-Ming Cheng} received his PhD degree from Tsinghua University in 2012,
supervised by Prof. Shi-Min Hu.
Then he did 2 years research fellow, with Prof. Philip Torr in Oxford.
He is now an associate professor at Nankai University.
His research interests includes computer graphics, computer vision, and image processing.
He has received the Google PhD fellowship award, the IBM PhD fellowship award, etc.
\end{IEEEbiography}
\vspace{-.45in}

\begin{IEEEbiography}[{\includegraphics[width=1in]{./Author/njm}}]
{Niloy J. Mitra} received his PhD degree from Stanford University in 2006.
He is currently a reader (associate professor) in
University College London~(UCL).
His research interests span a range of topics including shape analysis,
geometry processing, shape manipulation, image analysis, and recreational art.
He has received ACM SIGGRAPH `Significant New Researcher' award.
He is on the editorial boards of ACM TOG, Computer Graphics Forum, etc.
\end{IEEEbiography}
\vspace{-.45in}


\begin{IEEEbiography}[{\includegraphics[width=1in]{./Author/hxl}}]
{Xiaolei Huang} received her PhD degree from Rutgers University in 2006.
She is currently an assistant professor at Lehigh University.
Her research interests are in computer vision, computer graphics,
and multimedia retrieval.
Dr. Huang serves on the program committees of several international
conferences on computer vision and computer graphics,
and she reviews papers regularly for journals including the IEEE TPAMI,
and the IEEE TIP.
\end{IEEEbiography}
\vspace{-.45in}

\begin{IEEEbiography}[{\includegraphics[width=1in]{./Author/torr}}]
{Philip H. S. Torr} received the PhD degree from Oxford University.
After working for another three years at Oxford,
he worked for six years as a research scientist for Microsoft Research,
first in Redmond, then in Cambridge,
founding the vision side of the Machine Learning and Perception Group.
He is now a professor at Oxford University.
He has won awards from several top vision conferences,
including ICCV, CVPR, ECCV, NIPS and BMVC.
He is a Royal Society Wolfson Research Merit Award holder.
\end{IEEEbiography}
\vspace{-.4in}


\begin{IEEEbiography}[{\includegraphics[width=1in]{./Author/hsm}}]
{Shi-Min Hu} received the PhD degree from Zhejiang University in 1996.
He is currently a chair professor of computer science in the
Dept. of Computer Science and Technology, Tsinghua University, Beijing.
His research interests include digital geometry processing, video processing,
rendering, computer animation, and computer-aided geometric design.
He is an associate editor-in-chief of The Visual Computer,
and on the editorial board of IEEE TVCG, CAD, etc.
\end{IEEEbiography}

\vfill

% Can be used to pull up biographies so that the bottom of the last one
% is flush with the other column.
%\enlargethispage{-5in}

% that's all folks
\end{document}

